2024-11-21 13:23:44,675 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-21 13:23:44,691 main DEBUG Took 0.013284 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 13:23:44,691 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 13:23:44,692 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 13:23:44,693 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 13:23:44,694 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 13:23:44,703 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 13:23:44,720 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,723 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 13:23:44,725 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,725 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 13:23:44,726 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,727 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 13:23:44,728 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,729 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 13:23:44,729 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,730 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 13:23:44,731 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,732 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 13:23:44,733 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,734 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 13:23:44,734 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,735 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 13:23:44,736 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,736 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 13:23:44,737 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,737 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 13:23:44,738 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,738 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 13:23:44,739 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,739 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 13:23:44,740 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,741 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 13:23:44,742 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 13:23:44,744 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 13:23:44,747 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 13:23:44,748 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 13:23:44,750 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 13:23:44,750 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 13:23:44,762 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 13:23:44,766 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 13:23:44,768 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 13:23:44,769 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 13:23:44,769 main DEBUG createAppenders(={Console}) 2024-11-21 13:23:44,770 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-21 13:23:44,771 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-21 13:23:44,771 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-21 13:23:44,772 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 13:23:44,773 main DEBUG OutputStream closed 2024-11-21 13:23:44,773 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 13:23:44,774 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 13:23:44,774 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-21 13:23:44,871 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 13:23:44,875 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 13:23:44,876 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 13:23:44,878 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 13:23:44,879 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 13:23:44,879 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 13:23:44,880 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 13:23:44,880 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 13:23:44,881 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 13:23:44,881 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 13:23:44,882 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 13:23:44,882 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 13:23:44,882 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 13:23:44,883 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 13:23:44,883 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 13:23:44,883 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 13:23:44,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 13:23:44,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 13:23:44,887 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 13:23:44,887 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-21 13:23:44,888 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 13:23:44,889 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-21T13:23:45,145 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5 2024-11-21 13:23:45,150 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 13:23:45,150 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T13:23:45,160 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-21T13:23:45,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T13:23:45,204 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/cluster_cd6fb7cf-33aa-fbb2-dd15-3e00518ea929, deleteOnExit=true 2024-11-21T13:23:45,205 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T13:23:45,207 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/test.cache.data in system properties and HBase conf 2024-11-21T13:23:45,208 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T13:23:45,209 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/hadoop.log.dir in system properties and HBase conf 2024-11-21T13:23:45,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T13:23:45,210 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T13:23:45,211 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T13:23:45,300 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T13:23:45,392 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T13:23:45,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T13:23:45,396 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T13:23:45,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T13:23:45,397 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T13:23:45,398 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T13:23:45,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T13:23:45,399 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T13:23:45,400 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T13:23:45,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T13:23:45,401 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/nfs.dump.dir in system properties and HBase conf 2024-11-21T13:23:45,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/java.io.tmpdir in system properties and HBase conf 2024-11-21T13:23:45,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T13:23:45,403 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T13:23:45,404 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T13:23:46,678 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T13:23:46,751 INFO [Time-limited test {}] log.Log(170): Logging initialized @2973ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T13:23:46,821 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T13:23:46,885 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T13:23:46,914 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T13:23:46,914 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T13:23:46,916 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T13:23:46,936 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T13:23:46,940 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@610d6b27{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/hadoop.log.dir/,AVAILABLE} 2024-11-21T13:23:46,941 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@31a6a643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T13:23:47,180 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7f405bdd{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/java.io.tmpdir/jetty-localhost-40735-hadoop-hdfs-3_4_1-tests_jar-_-any-5512766345495879381/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T13:23:47,189 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5b829fcb{HTTP/1.1, (http/1.1)}{localhost:40735} 2024-11-21T13:23:47,190 INFO [Time-limited test {}] server.Server(415): Started @3412ms 2024-11-21T13:23:47,785 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T13:23:47,793 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T13:23:47,794 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T13:23:47,794 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T13:23:47,794 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T13:23:47,795 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@350067e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/hadoop.log.dir/,AVAILABLE} 2024-11-21T13:23:47,795 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@24457f8e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T13:23:47,899 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1e531827{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/java.io.tmpdir/jetty-localhost-39159-hadoop-hdfs-3_4_1-tests_jar-_-any-12903785432646081304/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T13:23:47,900 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4ea205b6{HTTP/1.1, (http/1.1)}{localhost:39159} 2024-11-21T13:23:47,900 INFO [Time-limited test {}] server.Server(415): Started @4122ms 2024-11-21T13:23:47,956 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T13:23:49,070 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/cluster_cd6fb7cf-33aa-fbb2-dd15-3e00518ea929/data/data1/current/BP-814699736-172.17.0.2-1732195426084/current, will proceed with Du for space computation calculation, 2024-11-21T13:23:49,070 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/cluster_cd6fb7cf-33aa-fbb2-dd15-3e00518ea929/data/data2/current/BP-814699736-172.17.0.2-1732195426084/current, will proceed with Du for space computation calculation, 2024-11-21T13:23:49,109 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T13:23:49,162 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaa793b851a997a02 with lease ID 0xc1e6415e0272be05: Processing first storage report for DS-3b483769-d442-48d4-86c2-6f68254f7dea from datanode DatanodeRegistration(127.0.0.1:44249, datanodeUuid=47d5bed9-0627-49b8-995c-9e5afce33aad, infoPort=45821, infoSecurePort=0, ipcPort=43963, storageInfo=lv=-57;cid=testClusterID;nsid=1836195786;c=1732195426084) 2024-11-21T13:23:49,164 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaa793b851a997a02 with lease ID 0xc1e6415e0272be05: from storage DS-3b483769-d442-48d4-86c2-6f68254f7dea node DatanodeRegistration(127.0.0.1:44249, datanodeUuid=47d5bed9-0627-49b8-995c-9e5afce33aad, infoPort=45821, infoSecurePort=0, ipcPort=43963, storageInfo=lv=-57;cid=testClusterID;nsid=1836195786;c=1732195426084), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-21T13:23:49,165 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xaa793b851a997a02 with lease ID 0xc1e6415e0272be05: Processing first storage report for DS-95ee18d2-e598-4c17-b6aa-d5d978dc33b6 from datanode DatanodeRegistration(127.0.0.1:44249, datanodeUuid=47d5bed9-0627-49b8-995c-9e5afce33aad, infoPort=45821, infoSecurePort=0, ipcPort=43963, storageInfo=lv=-57;cid=testClusterID;nsid=1836195786;c=1732195426084) 2024-11-21T13:23:49,166 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xaa793b851a997a02 with lease ID 0xc1e6415e0272be05: from storage DS-95ee18d2-e598-4c17-b6aa-d5d978dc33b6 node DatanodeRegistration(127.0.0.1:44249, datanodeUuid=47d5bed9-0627-49b8-995c-9e5afce33aad, infoPort=45821, infoSecurePort=0, ipcPort=43963, storageInfo=lv=-57;cid=testClusterID;nsid=1836195786;c=1732195426084), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T13:23:49,233 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5 2024-11-21T13:23:49,310 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/cluster_cd6fb7cf-33aa-fbb2-dd15-3e00518ea929/zookeeper_0, clientPort=59696, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/cluster_cd6fb7cf-33aa-fbb2-dd15-3e00518ea929/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/cluster_cd6fb7cf-33aa-fbb2-dd15-3e00518ea929/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T13:23:49,319 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59696 2024-11-21T13:23:49,330 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T13:23:49,332 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T13:23:49,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741825_1001 (size=7) 2024-11-21T13:23:49,953 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48 with version=8 2024-11-21T13:23:49,953 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/hbase-staging 2024-11-21T13:23:50,035 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T13:23:50,278 INFO [Time-limited test {}] client.ConnectionUtils(128): master/4fe6efa1e611:0 server-side Connection retries=45 2024-11-21T13:23:50,287 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T13:23:50,288 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T13:23:50,292 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T13:23:50,292 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T13:23:50,293 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T13:23:50,421 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T13:23:50,473 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T13:23:50,480 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T13:23:50,484 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T13:23:50,505 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 32639 (auto-detected) 2024-11-21T13:23:50,506 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T13:23:50,523 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46031 2024-11-21T13:23:50,543 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46031 connecting to ZooKeeper ensemble=127.0.0.1:59696 2024-11-21T13:23:50,653 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:460310x0, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T13:23:50,657 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46031-0x1015d90a7550000 connected 2024-11-21T13:23:50,737 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T13:23:50,739 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T13:23:50,747 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T13:23:50,750 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48, hbase.cluster.distributed=false 2024-11-21T13:23:50,772 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T13:23:50,777 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46031 2024-11-21T13:23:50,778 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46031 2024-11-21T13:23:50,780 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46031 2024-11-21T13:23:50,781 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46031 2024-11-21T13:23:50,782 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46031 2024-11-21T13:23:50,882 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/4fe6efa1e611:0 server-side Connection retries=45 2024-11-21T13:23:50,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T13:23:50,883 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T13:23:50,884 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T13:23:50,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T13:23:50,884 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T13:23:50,887 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T13:23:50,890 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T13:23:50,891 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33573 2024-11-21T13:23:50,893 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33573 connecting to ZooKeeper ensemble=127.0.0.1:59696 2024-11-21T13:23:50,894 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T13:23:50,896 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T13:23:50,904 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:335730x0, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T13:23:50,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33573-0x1015d90a7550001 connected 2024-11-21T13:23:50,905 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T13:23:50,909 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T13:23:50,918 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T13:23:50,920 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T13:23:50,925 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T13:23:50,928 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33573 2024-11-21T13:23:50,928 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33573 2024-11-21T13:23:50,929 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33573 2024-11-21T13:23:50,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33573 2024-11-21T13:23:50,930 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33573 2024-11-21T13:23:50,945 DEBUG [M:0;4fe6efa1e611:46031 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;4fe6efa1e611:46031 2024-11-21T13:23:50,946 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/4fe6efa1e611,46031,1732195430123 2024-11-21T13:23:50,957 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T13:23:50,957 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T13:23:50,959 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/4fe6efa1e611,46031,1732195430123 2024-11-21T13:23:50,988 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T13:23:50,988 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:23:50,988 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:23:50,989 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T13:23:50,991 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/4fe6efa1e611,46031,1732195430123 from backup master directory 2024-11-21T13:23:50,999 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T13:23:50,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/4fe6efa1e611,46031,1732195430123 2024-11-21T13:23:50,999 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T13:23:50,999 WARN [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T13:23:51,000 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=4fe6efa1e611,46031,1732195430123 2024-11-21T13:23:51,002 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T13:23:51,003 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T13:23:51,055 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/hbase.id] with ID: fe422165-ca34-4f60-b78c-7299a18fbab3 2024-11-21T13:23:51,055 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/.tmp/hbase.id 2024-11-21T13:23:51,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741826_1002 (size=42) 2024-11-21T13:23:51,468 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/.tmp/hbase.id]:[hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/hbase.id] 2024-11-21T13:23:51,514 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T13:23:51,519 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T13:23:51,538 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-21T13:23:51,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:23:51,546 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:23:51,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741827_1003 (size=196) 2024-11-21T13:23:51,985 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T13:23:51,986 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T13:23:52,000 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:52,005 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T13:23:52,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741828_1004 (size=1189) 2024-11-21T13:23:52,450 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store 2024-11-21T13:23:52,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741829_1005 (size=34) 2024-11-21T13:23:52,882 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T13:23:52,885 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:23:52,887 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T13:23:52,887 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T13:23:52,887 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T13:23:52,888 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T13:23:52,889 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T13:23:52,889 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T13:23:52,890 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732195432886Disabling compacts and flushes for region at 1732195432886Disabling writes for close at 1732195432889 (+3 ms)Writing region close event to WAL at 1732195432889Closed at 1732195432889 2024-11-21T13:23:52,893 WARN [master/4fe6efa1e611:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/.initializing 2024-11-21T13:23:52,893 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/WALs/4fe6efa1e611,46031,1732195430123 2024-11-21T13:23:52,901 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T13:23:52,916 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4fe6efa1e611%2C46031%2C1732195430123, suffix=, logDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/WALs/4fe6efa1e611,46031,1732195430123, archiveDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/oldWALs, maxLogs=10 2024-11-21T13:23:52,937 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/WALs/4fe6efa1e611,46031,1732195430123/4fe6efa1e611%2C46031%2C1732195430123.1732195432920, exclude list is [], retry=0 2024-11-21T13:23:52,952 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44249,DS-3b483769-d442-48d4-86c2-6f68254f7dea,DISK] 2024-11-21T13:23:52,955 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T13:23:52,988 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/WALs/4fe6efa1e611,46031,1732195430123/4fe6efa1e611%2C46031%2C1732195430123.1732195432920 2024-11-21T13:23:52,989 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45821:45821)] 2024-11-21T13:23:52,990 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T13:23:52,990 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:23:52,993 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T13:23:52,994 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T13:23:53,030 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T13:23:53,052 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T13:23:53,055 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:53,058 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T13:23:53,058 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T13:23:53,061 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T13:23:53,062 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:53,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:23:53,063 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T13:23:53,066 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T13:23:53,066 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:53,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:23:53,068 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T13:23:53,071 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T13:23:53,071 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:53,072 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:23:53,073 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T13:23:53,076 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T13:23:53,077 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T13:23:53,081 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T13:23:53,082 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T13:23:53,085 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T13:23:53,089 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T13:23:53,093 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T13:23:53,095 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71122934, jitterRate=0.05981430411338806}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T13:23:53,101 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732195433007Initializing all the Stores at 1732195433009 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732195433010 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195433011 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195433011Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195433011Cleaning up temporary data from old regions at 1732195433082 (+71 ms)Region opened successfully at 1732195433101 (+19 ms) 2024-11-21T13:23:53,103 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T13:23:53,132 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@464d2f4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4fe6efa1e611/172.17.0.2:0 2024-11-21T13:23:53,161 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T13:23:53,172 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-21T13:23:53,172 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-21T13:23:53,175 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T13:23:53,176 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T13:23:53,181 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T13:23:53,181 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T13:23:53,210 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T13:23:53,220 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T13:23:53,241 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T13:23:53,243 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T13:23:53,245 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T13:23:53,251 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T13:23:53,253 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T13:23:53,257 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T13:23:53,261 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T13:23:53,263 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T13:23:53,272 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T13:23:53,295 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T13:23:53,303 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T13:23:53,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T13:23:53,314 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T13:23:53,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:23:53,314 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:23:53,318 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=4fe6efa1e611,46031,1732195430123, sessionid=0x1015d90a7550000, setting cluster-up flag (Was=false) 2024-11-21T13:23:53,346 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:23:53,346 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:23:53,377 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T13:23:53,381 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4fe6efa1e611,46031,1732195430123 2024-11-21T13:23:53,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:23:53,398 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:23:53,430 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T13:23:53,432 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=4fe6efa1e611,46031,1732195430123 2024-11-21T13:23:53,437 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T13:23:53,516 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T13:23:53,526 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T13:23:53,534 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T13:23:53,538 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(746): ClusterId : fe422165-ca34-4f60-b78c-7299a18fbab3 2024-11-21T13:23:53,541 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T13:23:53,542 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 4fe6efa1e611,46031,1732195430123 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T13:23:53,559 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T13:23:53,559 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T13:23:53,559 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/4fe6efa1e611:0, corePoolSize=5, maxPoolSize=5 2024-11-21T13:23:53,560 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/4fe6efa1e611:0, corePoolSize=5, maxPoolSize=5 2024-11-21T13:23:53,560 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/4fe6efa1e611:0, corePoolSize=5, maxPoolSize=5 2024-11-21T13:23:53,560 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/4fe6efa1e611:0, corePoolSize=5, maxPoolSize=5 2024-11-21T13:23:53,560 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/4fe6efa1e611:0, corePoolSize=10, maxPoolSize=10 2024-11-21T13:23:53,560 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/4fe6efa1e611:0, corePoolSize=1, maxPoolSize=1 2024-11-21T13:23:53,561 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/4fe6efa1e611:0, corePoolSize=2, maxPoolSize=2 2024-11-21T13:23:53,561 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/4fe6efa1e611:0, corePoolSize=1, maxPoolSize=1 2024-11-21T13:23:53,565 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732195463565 2024-11-21T13:23:53,567 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T13:23:53,568 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T13:23:53,569 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T13:23:53,570 DEBUG [RS:0;4fe6efa1e611:33573 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7cf4ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=4fe6efa1e611/172.17.0.2:0 2024-11-21T13:23:53,570 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T13:23:53,571 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T13:23:53,573 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T13:23:53,573 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T13:23:53,574 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T13:23:53,574 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T13:23:53,575 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,584 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:53,585 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T13:23:53,589 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T13:23:53,590 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;4fe6efa1e611:33573 2024-11-21T13:23:53,591 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T13:23:53,591 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T13:23:53,594 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T13:23:53,594 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T13:23:53,594 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T13:23:53,597 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(2659): reportForDuty to master=4fe6efa1e611,46031,1732195430123 with port=33573, startcode=1732195430851 2024-11-21T13:23:53,597 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T13:23:53,598 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T13:23:53,604 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/4fe6efa1e611:0:becomeActiveMaster-HFileCleaner.large.0-1732195433600,5,FailOnTimeoutGroup] 2024-11-21T13:23:53,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741831_1007 (size=1321) 2024-11-21T13:23:53,611 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T13:23:53,612 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48 2024-11-21T13:23:53,613 DEBUG [RS:0;4fe6efa1e611:33573 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T13:23:53,616 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/4fe6efa1e611:0:becomeActiveMaster-HFileCleaner.small.0-1732195433604,5,FailOnTimeoutGroup] 2024-11-21T13:23:53,617 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,617 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T13:23:53,618 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,619 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741832_1008 (size=32) 2024-11-21T13:23:53,664 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:23:53,676 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T13:23:53,678 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36177, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T13:23:53,683 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T13:23:53,683 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:53,686 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T13:23:53,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T13:23:53,687 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46031 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:53,691 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46031 {}] master.ServerManager(517): Registering regionserver=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:53,701 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T13:23:53,701 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:53,705 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T13:23:53,706 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T13:23:53,711 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T13:23:53,711 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:53,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T13:23:53,713 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T13:23:53,716 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48 2024-11-21T13:23:53,716 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39169 2024-11-21T13:23:53,717 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T13:23:53,717 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T13:23:53,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:53,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T13:23:53,719 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T13:23:53,721 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740 2024-11-21T13:23:53,723 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740 2024-11-21T13:23:53,727 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T13:23:53,727 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T13:23:53,729 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T13:23:53,732 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T13:23:53,740 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T13:23:53,742 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73448526, jitterRate=0.09446832537651062}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T13:23:53,746 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T13:23:53,746 DEBUG [RS:0;4fe6efa1e611:33573 {}] zookeeper.ZKUtil(111): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:53,746 WARN [RS:0;4fe6efa1e611:33573 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T13:23:53,747 INFO [RS:0;4fe6efa1e611:33573 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T13:23:53,747 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/WALs/4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:53,747 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732195433665Initializing all the Stores at 1732195433669 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732195433670 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732195433673 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195433673Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732195433673Cleaning up temporary data from old regions at 1732195433727 (+54 ms)Region opened successfully at 1732195433747 (+20 ms) 2024-11-21T13:23:53,747 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T13:23:53,748 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T13:23:53,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T13:23:53,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T13:23:53,748 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T13:23:53,751 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [4fe6efa1e611,33573,1732195430851] 2024-11-21T13:23:53,753 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T13:23:53,753 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732195433747Disabling compacts and flushes for region at 1732195433747Disabling writes for close at 1732195433748 (+1 ms)Writing region close event to WAL at 1732195433753 (+5 ms)Closed at 1732195433753 2024-11-21T13:23:53,757 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T13:23:53,757 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T13:23:53,765 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T13:23:53,773 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T13:23:53,778 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T13:23:53,779 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T13:23:53,801 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T13:23:53,810 INFO [RS:0;4fe6efa1e611:33573 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T13:23:53,810 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,811 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T13:23:53,819 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T13:23:53,822 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,822 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/4fe6efa1e611:0, corePoolSize=1, maxPoolSize=1 2024-11-21T13:23:53,822 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/4fe6efa1e611:0, corePoolSize=1, maxPoolSize=1 2024-11-21T13:23:53,823 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/4fe6efa1e611:0, corePoolSize=1, maxPoolSize=1 2024-11-21T13:23:53,823 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/4fe6efa1e611:0, corePoolSize=1, maxPoolSize=1 2024-11-21T13:23:53,823 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/4fe6efa1e611:0, corePoolSize=1, maxPoolSize=1 2024-11-21T13:23:53,823 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/4fe6efa1e611:0, corePoolSize=2, maxPoolSize=2 2024-11-21T13:23:53,823 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/4fe6efa1e611:0, corePoolSize=1, maxPoolSize=1 2024-11-21T13:23:53,824 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/4fe6efa1e611:0, corePoolSize=1, maxPoolSize=1 2024-11-21T13:23:53,824 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/4fe6efa1e611:0, corePoolSize=1, maxPoolSize=1 2024-11-21T13:23:53,824 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/4fe6efa1e611:0, corePoolSize=1, maxPoolSize=1 2024-11-21T13:23:53,824 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/4fe6efa1e611:0, corePoolSize=1, maxPoolSize=1 2024-11-21T13:23:53,824 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/4fe6efa1e611:0, corePoolSize=1, maxPoolSize=1 2024-11-21T13:23:53,825 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/4fe6efa1e611:0, corePoolSize=3, maxPoolSize=3 2024-11-21T13:23:53,825 DEBUG [RS:0;4fe6efa1e611:33573 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0, corePoolSize=3, maxPoolSize=3 2024-11-21T13:23:53,832 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,833 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,833 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,834 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,834 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,836 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.ChoreService(168): Chore ScheduledChore name=4fe6efa1e611,33573,1732195430851-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T13:23:53,856 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T13:23:53,858 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.ChoreService(168): Chore ScheduledChore name=4fe6efa1e611,33573,1732195430851-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,859 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,859 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.Replication(171): 4fe6efa1e611,33573,1732195430851 started 2024-11-21T13:23:53,879 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:53,880 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1482): Serving as 4fe6efa1e611,33573,1732195430851, RpcServer on 4fe6efa1e611/172.17.0.2:33573, sessionid=0x1015d90a7550001 2024-11-21T13:23:53,881 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T13:23:53,881 DEBUG [RS:0;4fe6efa1e611:33573 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:53,882 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4fe6efa1e611,33573,1732195430851' 2024-11-21T13:23:53,882 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T13:23:53,883 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T13:23:53,884 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T13:23:53,885 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T13:23:53,885 DEBUG [RS:0;4fe6efa1e611:33573 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:53,885 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '4fe6efa1e611,33573,1732195430851' 2024-11-21T13:23:53,896 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T13:23:53,897 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T13:23:53,898 DEBUG [RS:0;4fe6efa1e611:33573 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T13:23:53,898 INFO [RS:0;4fe6efa1e611:33573 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T13:23:53,898 INFO [RS:0;4fe6efa1e611:33573 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T13:23:53,930 WARN [4fe6efa1e611:46031 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T13:23:54,003 INFO [RS:0;4fe6efa1e611:33573 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T13:23:54,007 INFO [RS:0;4fe6efa1e611:33573 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4fe6efa1e611%2C33573%2C1732195430851, suffix=, logDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/WALs/4fe6efa1e611,33573,1732195430851, archiveDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/oldWALs, maxLogs=32 2024-11-21T13:23:54,025 DEBUG [RS:0;4fe6efa1e611:33573 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/WALs/4fe6efa1e611,33573,1732195430851/4fe6efa1e611%2C33573%2C1732195430851.1732195434010, exclude list is [], retry=0 2024-11-21T13:23:54,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44249,DS-3b483769-d442-48d4-86c2-6f68254f7dea,DISK] 2024-11-21T13:23:54,038 INFO [RS:0;4fe6efa1e611:33573 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/WALs/4fe6efa1e611,33573,1732195430851/4fe6efa1e611%2C33573%2C1732195430851.1732195434010 2024-11-21T13:23:54,038 DEBUG [RS:0;4fe6efa1e611:33573 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45821:45821)] 2024-11-21T13:23:54,183 DEBUG [4fe6efa1e611:46031 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-21T13:23:54,194 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:54,201 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fe6efa1e611,33573,1732195430851, state=OPENING 2024-11-21T13:23:54,251 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T13:23:54,261 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:23:54,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:23:54,263 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T13:23:54,264 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T13:23:54,264 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T13:23:54,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:23:54,447 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T13:23:54,453 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57977, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T13:23:54,468 INFO [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T13:23:54,469 INFO [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T13:23:54,469 INFO [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T13:23:54,479 INFO [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=4fe6efa1e611%2C33573%2C1732195430851.meta, suffix=.meta, logDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/WALs/4fe6efa1e611,33573,1732195430851, archiveDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/oldWALs, maxLogs=32 2024-11-21T13:23:54,501 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/WALs/4fe6efa1e611,33573,1732195430851/4fe6efa1e611%2C33573%2C1732195430851.meta.1732195434481.meta, exclude list is [], retry=0 2024-11-21T13:23:54,506 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:44249,DS-3b483769-d442-48d4-86c2-6f68254f7dea,DISK] 2024-11-21T13:23:54,510 INFO [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/WALs/4fe6efa1e611,33573,1732195430851/4fe6efa1e611%2C33573%2C1732195430851.meta.1732195434481.meta 2024-11-21T13:23:54,511 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:45821:45821)] 2024-11-21T13:23:54,512 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T13:23:54,514 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T13:23:54,518 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T13:23:54,524 INFO [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T13:23:54,530 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T13:23:54,531 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:23:54,531 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T13:23:54,531 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T13:23:54,535 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T13:23:54,538 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T13:23:54,538 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:54,539 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T13:23:54,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T13:23:54,542 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T13:23:54,543 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:54,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T13:23:54,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T13:23:54,546 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T13:23:54,547 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:54,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T13:23:54,548 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T13:23:54,550 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T13:23:54,550 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:54,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T13:23:54,552 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T13:23:54,554 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740 2024-11-21T13:23:54,558 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740 2024-11-21T13:23:54,561 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T13:23:54,561 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T13:23:54,563 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T13:23:54,567 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T13:23:54,570 INFO [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71202227, jitterRate=0.060995861887931824}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T13:23:54,571 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T13:23:54,573 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732195434532Writing region info on filesystem at 1732195434532Initializing all the Stores at 1732195434534 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732195434535 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732195434535Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195434535Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732195434535Cleaning up temporary data from old regions at 1732195434562 (+27 ms)Running coprocessor post-open hooks at 1732195434571 (+9 ms)Region opened successfully at 1732195434572 (+1 ms) 2024-11-21T13:23:54,583 INFO [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732195434435 2024-11-21T13:23:54,599 DEBUG [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T13:23:54,599 INFO [RS_OPEN_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T13:23:54,602 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:54,605 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 4fe6efa1e611,33573,1732195430851, state=OPEN 2024-11-21T13:23:54,723 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T13:23:54,723 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T13:23:54,723 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T13:23:54,723 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T13:23:54,724 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:54,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T13:23:54,734 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=4fe6efa1e611,33573,1732195430851 in 458 msec 2024-11-21T13:23:54,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T13:23:54,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 971 msec 2024-11-21T13:23:54,744 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T13:23:54,744 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T13:23:54,769 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:23:54,770 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:23:54,788 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:23:54,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54845, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:23:54,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3530 sec 2024-11-21T13:23:54,820 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732195434820, completionTime=-1 2024-11-21T13:23:54,822 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-21T13:23:54,822 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T13:23:54,850 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-21T13:23:54,850 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732195494850 2024-11-21T13:23:54,850 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732195554850 2024-11-21T13:23:54,851 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 28 msec 2024-11-21T13:23:54,853 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4fe6efa1e611,46031,1732195430123-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:54,853 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4fe6efa1e611,46031,1732195430123-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:54,853 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4fe6efa1e611,46031,1732195430123-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:54,854 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-4fe6efa1e611:46031, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:54,855 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:54,855 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:54,862 DEBUG [master/4fe6efa1e611:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T13:23:54,890 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.890sec 2024-11-21T13:23:54,892 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T13:23:54,893 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T13:23:54,895 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T13:23:54,896 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T13:23:54,896 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T13:23:54,897 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4fe6efa1e611,46031,1732195430123-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T13:23:54,898 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4fe6efa1e611,46031,1732195430123-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T13:23:54,908 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T13:23:54,909 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T13:23:54,909 INFO [master/4fe6efa1e611:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=4fe6efa1e611,46031,1732195430123-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T13:23:54,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63492372, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:54,953 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T13:23:54,953 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T13:23:54,957 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:23:54,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:23:54,970 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:23:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:23:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:23:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69d1ff91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:23:54,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:23:54,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:23:54,982 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56308, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:23:54,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d4ae4df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:54,985 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:23:54,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:23:54,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:23:54,998 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:23:55,019 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=4fe6efa1e611,46031,1732195430123 2024-11-21T13:23:55,031 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=461, ProcessCount=11, AvailableMemoryMB=3793 2024-11-21T13:23:55,061 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T13:23:55,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 4fe6efa1e611,46031,1732195430123 2024-11-21T13:23:55,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33f36750 2024-11-21T13:23:55,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T13:23:55,076 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56320, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T13:23:55,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T13:23:55,083 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T13:23:55,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T13:23:55,101 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T13:23:55,103 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-21T13:23:55,104 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:55,107 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T13:23:55,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T13:23:55,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741835_1011 (size=959) 2024-11-21T13:23:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T13:23:55,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T13:23:55,538 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48 2024-11-21T13:23:55,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741836_1012 (size=53) 2024-11-21T13:23:55,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:23:55,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 78998c23688e9623a40dfb78a0d6fff8, disabling compactions & flushes 2024-11-21T13:23:55,549 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:55,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:55,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. after waiting 0 ms 2024-11-21T13:23:55,549 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:55,549 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:55,550 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 78998c23688e9623a40dfb78a0d6fff8: Waiting for close lock at 1732195435549Disabling compacts and flushes for region at 1732195435549Disabling writes for close at 1732195435549Writing region close event to WAL at 1732195435549Closed at 1732195435549 2024-11-21T13:23:55,552 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T13:23:55,556 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732195435552"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732195435552"}]},"ts":"1732195435552"} 2024-11-21T13:23:55,561 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T13:23:55,563 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T13:23:55,565 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195435563"}]},"ts":"1732195435563"} 2024-11-21T13:23:55,569 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T13:23:55,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78998c23688e9623a40dfb78a0d6fff8, ASSIGN}] 2024-11-21T13:23:55,573 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78998c23688e9623a40dfb78a0d6fff8, ASSIGN 2024-11-21T13:23:55,575 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78998c23688e9623a40dfb78a0d6fff8, ASSIGN; state=OFFLINE, location=4fe6efa1e611,33573,1732195430851; forceNewPlan=false, retain=false 2024-11-21T13:23:55,726 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=78998c23688e9623a40dfb78a0d6fff8, regionState=OPENING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:55,732 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78998c23688e9623a40dfb78a0d6fff8, ASSIGN because future has completed 2024-11-21T13:23:55,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:23:55,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T13:23:55,896 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:55,896 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} 2024-11-21T13:23:55,897 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:55,897 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:23:55,897 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:55,897 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:55,900 INFO [StoreOpener-78998c23688e9623a40dfb78a0d6fff8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:55,902 INFO [StoreOpener-78998c23688e9623a40dfb78a0d6fff8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78998c23688e9623a40dfb78a0d6fff8 columnFamilyName A 2024-11-21T13:23:55,902 DEBUG [StoreOpener-78998c23688e9623a40dfb78a0d6fff8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:55,904 INFO [StoreOpener-78998c23688e9623a40dfb78a0d6fff8-1 {}] regionserver.HStore(327): Store=78998c23688e9623a40dfb78a0d6fff8/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:23:55,904 INFO [StoreOpener-78998c23688e9623a40dfb78a0d6fff8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:55,907 INFO [StoreOpener-78998c23688e9623a40dfb78a0d6fff8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78998c23688e9623a40dfb78a0d6fff8 columnFamilyName B 2024-11-21T13:23:55,907 DEBUG [StoreOpener-78998c23688e9623a40dfb78a0d6fff8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:55,908 INFO [StoreOpener-78998c23688e9623a40dfb78a0d6fff8-1 {}] regionserver.HStore(327): Store=78998c23688e9623a40dfb78a0d6fff8/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:23:55,908 INFO [StoreOpener-78998c23688e9623a40dfb78a0d6fff8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:55,911 INFO [StoreOpener-78998c23688e9623a40dfb78a0d6fff8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 78998c23688e9623a40dfb78a0d6fff8 columnFamilyName C 2024-11-21T13:23:55,912 DEBUG [StoreOpener-78998c23688e9623a40dfb78a0d6fff8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:23:55,913 INFO [StoreOpener-78998c23688e9623a40dfb78a0d6fff8-1 {}] regionserver.HStore(327): Store=78998c23688e9623a40dfb78a0d6fff8/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:23:55,913 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:55,915 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:55,916 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:55,919 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:55,919 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:55,920 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T13:23:55,924 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:55,932 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T13:23:55,934 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 78998c23688e9623a40dfb78a0d6fff8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72374331, jitterRate=0.07846157252788544}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T13:23:55,934 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:55,936 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 78998c23688e9623a40dfb78a0d6fff8: Running coprocessor pre-open hook at 1732195435897Writing region info on filesystem at 1732195435897Initializing all the Stores at 1732195435899 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195435899Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195435900 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195435900Cleaning up temporary data from old regions at 1732195435919 (+19 ms)Running coprocessor post-open hooks at 1732195435934 (+15 ms)Region opened successfully at 1732195435935 (+1 ms) 2024-11-21T13:23:55,938 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., pid=6, masterSystemTime=1732195435888 2024-11-21T13:23:55,943 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:55,943 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:55,945 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=78998c23688e9623a40dfb78a0d6fff8, regionState=OPEN, openSeqNum=2, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:55,950 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:23:55,952 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46031 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=4fe6efa1e611,33573,1732195430851, table=TestAcidGuarantees, region=78998c23688e9623a40dfb78a0d6fff8. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T13:23:55,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-21T13:23:55,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 in 221 msec 2024-11-21T13:23:55,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T13:23:55,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78998c23688e9623a40dfb78a0d6fff8, ASSIGN in 389 msec 2024-11-21T13:23:55,970 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T13:23:55,970 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195435970"}]},"ts":"1732195435970"} 2024-11-21T13:23:55,975 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T13:23:55,978 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T13:23:55,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 891 msec 2024-11-21T13:23:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T13:23:56,251 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:23:56,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45979b05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,259 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:23:56,259 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:23:56,261 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:23:56,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:23:56,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:23:56,262 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33c6058d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:23:56,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:23:56,265 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:23:56,267 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56326, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:23:56,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78e5dcf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,270 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:23:56,273 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:23:56,274 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:23:56,279 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:23:56,282 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T13:23:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4fe6efa1e611,46031,1732195430123 2024-11-21T13:23:56,284 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@10ab891d 2024-11-21T13:23:56,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T13:23:56,289 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T13:23:56,294 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f50427a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,294 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:23:56,295 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:23:56,297 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:23:56,297 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:23:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:23:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14496fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:23:56,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:23:56,300 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:23:56,302 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:23:56,303 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e49c5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@308ded33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,305 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:23:56,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:23:56,318 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:23:56,319 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:23:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:23:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7290076, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:23:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:23:56,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:23:56,323 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:23:56,324 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b17c100, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e41ab17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:23:56,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:23:56,327 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:23:56,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:23:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:23:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3687d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:23:56,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:23:56,332 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:23:56,334 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:23:56,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b383f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d30157, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,337 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:23:56,337 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:23:56,340 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:23:56,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:23:56,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:23:56,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@175526a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,341 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:23:56,342 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:23:56,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:23:56,346 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56418, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:23:56,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ede423c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9a72b63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:23:56,351 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:23:56,352 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:23:56,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:23:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:23:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@371cb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:23:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:23:56,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:23:56,355 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56432, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:23:56,356 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500e7ebc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209f7c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,361 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:23:56,361 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:23:56,362 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:23:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:23:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:23:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14983026, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:23:56,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:23:56,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:23:56,365 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:23:56,366 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12ee6c39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d1075e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,367 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:23:56,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:23:56,369 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:23:56,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:23:56,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:23:56,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18ee4412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:23:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:23:56,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:23:56,371 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:23:56,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c3b3fdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c345e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,373 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:23:56,373 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:23:56,374 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:23:56,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:23:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:23:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6e851d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:23:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:23:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:23:56,376 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:23:56,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29434e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28ccc0d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,378 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:23:56,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:23:56,380 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:23:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:23:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:23:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68732d4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,380 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:23:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:23:56,381 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:23:56,382 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56506, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:23:56,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ce716ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:23:56,387 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:23:56,388 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:23:56,388 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:23:56,389 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:23:56,390 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:23:56,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:23:56,390 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:23:56,390 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:23:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:23:56,391 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:23:56,391 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:23:56,392 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:23:56,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:23:56,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:23:56,393 DEBUG [Thread-156 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:23:56,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:23:56,394 DEBUG [Thread-156 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:23:56,396 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56372, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:23:56,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:23:56,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:23:56,399 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56374, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:23:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:23:56,399 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:23:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:23:56,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:23:56,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56378, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:23:56,401 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:23:56,402 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:23:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:23:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:23:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:23:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-21T13:23:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:23:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:23:56,407 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56388, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:23:56,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:23:56,409 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:23:56,410 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:23:56,412 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:23:56,413 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:23:56,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T13:23:56,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:23:56,415 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:23:56,417 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:23:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:23:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:23:56,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56400, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:23:56,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:23:56,421 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:23:56,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:23:56,426 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:23:56,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:56,494 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:23:56,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T13:23:56,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T13:23:56,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T13:23:56,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:56,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:23:56,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:56,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:56,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:56,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:56,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195496632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195496631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:56,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195496637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:56,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195496643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:56,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195496649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:56,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/f0bee866bbf346ee82dee66fbec8ccee is 50, key is test_row_0/A:col10/1732195436479/Put/seqid=0 2024-11-21T13:23:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,694 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,696 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:56,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741837_1013 (size=12001) 2024-11-21T13:23:56,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/f0bee866bbf346ee82dee66fbec8ccee 2024-11-21T13:23:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T13:23:56,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T13:23:56,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195496800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:56,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:56,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195496801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:56,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195496802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:56,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195496802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:56,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:56,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:23:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:56,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:56,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195496812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:56,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:56,823 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:56,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/b263dc6c504e4248a0db3001d8ee9e5e is 50, key is test_row_0/B:col10/1732195436479/Put/seqid=0 2024-11-21T13:23:56,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741838_1014 (size=12001) 2024-11-21T13:23:56,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/b263dc6c504e4248a0db3001d8ee9e5e 2024-11-21T13:23:56,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/c1ba87c9e4954cdb804ec2d34fb71310 is 50, key is test_row_0/C:col10/1732195436479/Put/seqid=0 2024-11-21T13:23:56,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T13:23:56,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:56,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:23:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:56,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:56,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:56,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741839_1015 (size=12001) 2024-11-21T13:23:57,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195497010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195497022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195497022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195497022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195497030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,033 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T13:23:57,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T13:23:57,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:57,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:23:57,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:57,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:57,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:57,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T13:23:57,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:57,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:23:57,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:57,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:57,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:57,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195497320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195497342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,348 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,349 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195497342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195497342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195497341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,392 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/c1ba87c9e4954cdb804ec2d34fb71310 2024-11-21T13:23:57,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/f0bee866bbf346ee82dee66fbec8ccee as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/f0bee866bbf346ee82dee66fbec8ccee 2024-11-21T13:23:57,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/f0bee866bbf346ee82dee66fbec8ccee, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T13:23:57,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/b263dc6c504e4248a0db3001d8ee9e5e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/b263dc6c504e4248a0db3001d8ee9e5e 2024-11-21T13:23:57,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T13:23:57,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:57,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:23:57,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:57,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:57,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:23:57,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/b263dc6c504e4248a0db3001d8ee9e5e, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T13:23:57,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/c1ba87c9e4954cdb804ec2d34fb71310 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/c1ba87c9e4954cdb804ec2d34fb71310 2024-11-21T13:23:57,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/c1ba87c9e4954cdb804ec2d34fb71310, entries=150, sequenceid=14, filesize=11.7 K 2024-11-21T13:23:57,492 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 78998c23688e9623a40dfb78a0d6fff8 in 983ms, sequenceid=14, compaction requested=false 2024-11-21T13:23:57,494 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T13:23:57,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:23:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T13:23:57,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-21T13:23:57,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:57,603 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T13:23:57,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/e505b33da07a412b8d9f053622c5f0e3 is 50, key is test_row_0/A:col10/1732195436638/Put/seqid=0 2024-11-21T13:23:57,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741840_1016 (size=12001) 2024-11-21T13:23:57,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:23:57,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:57,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195497890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,908 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195497896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195497893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195497904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,912 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:57,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:57,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195497907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195498021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195498022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,029 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195498022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195498023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195498024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,062 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/e505b33da07a412b8d9f053622c5f0e3 2024-11-21T13:23:58,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/bf37ed443e134eb4a44fda493468514c is 50, key is test_row_0/B:col10/1732195436638/Put/seqid=0 2024-11-21T13:23:58,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741841_1017 (size=12001) 2024-11-21T13:23:58,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195498240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195498241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195498241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195498241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195498244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,546 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/bf37ed443e134eb4a44fda493468514c 2024-11-21T13:23:58,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195498550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T13:23:58,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195498561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195498561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,567 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195498561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195498562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:58,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:58,570 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:58,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/4534e7cc58164897b98d56a4dab94737 is 50, key is test_row_0/C:col10/1732195436638/Put/seqid=0 2024-11-21T13:23:58,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741842_1018 (size=12001) 2024-11-21T13:23:58,637 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/4534e7cc58164897b98d56a4dab94737 2024-11-21T13:23:58,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/e505b33da07a412b8d9f053622c5f0e3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/e505b33da07a412b8d9f053622c5f0e3 2024-11-21T13:23:58,667 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/e505b33da07a412b8d9f053622c5f0e3, entries=150, sequenceid=38, filesize=11.7 K 2024-11-21T13:23:58,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/bf37ed443e134eb4a44fda493468514c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bf37ed443e134eb4a44fda493468514c 2024-11-21T13:23:58,690 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bf37ed443e134eb4a44fda493468514c, entries=150, sequenceid=38, filesize=11.7 K 2024-11-21T13:23:58,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/4534e7cc58164897b98d56a4dab94737 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/4534e7cc58164897b98d56a4dab94737 2024-11-21T13:23:58,714 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/4534e7cc58164897b98d56a4dab94737, entries=150, sequenceid=38, filesize=11.7 K 2024-11-21T13:23:58,717 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 78998c23688e9623a40dfb78a0d6fff8 in 1114ms, sequenceid=38, compaction requested=false 2024-11-21T13:23:58,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:23:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:23:58,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-21T13:23:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-21T13:23:58,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-21T13:23:58,727 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3020 sec 2024-11-21T13:23:58,738 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.3390 sec 2024-11-21T13:23:59,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:23:59,063 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:23:59,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T13:23:59,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/3c522fa5413243bda6f9a14a70f94e6a is 50, key is test_row_0/A:col10/1732195439060/Put/seqid=0 2024-11-21T13:23:59,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741843_1019 (size=12001) 2024-11-21T13:23:59,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195499138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195499147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195499151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195499153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195499156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,169 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195499260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195499260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195499270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195499270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195499280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,473 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195499470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,474 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195499479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195499479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195499480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195499489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,493 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/3c522fa5413243bda6f9a14a70f94e6a 2024-11-21T13:23:59,545 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/b1aa6a7e0c60480d9ac49a343dd01458 is 50, key is test_row_0/B:col10/1732195439060/Put/seqid=0 2024-11-21T13:23:59,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741844_1020 (size=12001) 2024-11-21T13:23:59,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/b1aa6a7e0c60480d9ac49a343dd01458 2024-11-21T13:23:59,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/b402ef22736549cf9e8e04ce513a1a29 is 50, key is test_row_0/C:col10/1732195439060/Put/seqid=0 2024-11-21T13:23:59,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741845_1021 (size=12001) 2024-11-21T13:23:59,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195499779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195499789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195499790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195499790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,800 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:23:59,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:23:59,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195499799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:23:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:23:59,801 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/b402ef22736549cf9e8e04ce513a1a29 2024-11-21T13:24:00,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/3c522fa5413243bda6f9a14a70f94e6a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/3c522fa5413243bda6f9a14a70f94e6a 2024-11-21T13:24:00,049 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T13:24:00,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/3c522fa5413243bda6f9a14a70f94e6a, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T13:24:00,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/b1aa6a7e0c60480d9ac49a343dd01458 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/b1aa6a7e0c60480d9ac49a343dd01458 2024-11-21T13:24:00,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/b1aa6a7e0c60480d9ac49a343dd01458, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T13:24:00,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/b402ef22736549cf9e8e04ce513a1a29 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b402ef22736549cf9e8e04ce513a1a29 2024-11-21T13:24:00,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b402ef22736549cf9e8e04ce513a1a29, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T13:24:00,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 78998c23688e9623a40dfb78a0d6fff8 in 1074ms, sequenceid=51, compaction requested=true 2024-11-21T13:24:00,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:00,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:00,166 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:00,170 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:00,171 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/A is initiating minor compaction (all files) 2024-11-21T13:24:00,172 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/A in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:00,173 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/f0bee866bbf346ee82dee66fbec8ccee, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/e505b33da07a412b8d9f053622c5f0e3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/3c522fa5413243bda6f9a14a70f94e6a] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=35.2 K 2024-11-21T13:24:00,175 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:00,176 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0bee866bbf346ee82dee66fbec8ccee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732195436415 2024-11-21T13:24:00,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:00,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:00,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:00,176 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T13:24:00,177 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting e505b33da07a412b8d9f053622c5f0e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732195436624 2024-11-21T13:24:00,178 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c522fa5413243bda6f9a14a70f94e6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732195437881 2024-11-21T13:24:00,182 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:00,186 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:00,186 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/B is initiating minor compaction (all files) 2024-11-21T13:24:00,186 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/B in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:00,186 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/b263dc6c504e4248a0db3001d8ee9e5e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bf37ed443e134eb4a44fda493468514c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/b1aa6a7e0c60480d9ac49a343dd01458] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=35.2 K 2024-11-21T13:24:00,193 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b263dc6c504e4248a0db3001d8ee9e5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732195436415 2024-11-21T13:24:00,194 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting bf37ed443e134eb4a44fda493468514c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732195436624 2024-11-21T13:24:00,197 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b1aa6a7e0c60480d9ac49a343dd01458, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732195437881 2024-11-21T13:24:00,202 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T13:24:00,267 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#A#compaction#9 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:00,267 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#B#compaction#10 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:00,270 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/06928b2608704771bc06c9ca84fdf92f is 50, key is test_row_0/A:col10/1732195439060/Put/seqid=0 2024-11-21T13:24:00,270 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/66099bbe30d0468da4df28f9dd8a9684 is 50, key is test_row_0/B:col10/1732195439060/Put/seqid=0 2024-11-21T13:24:00,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741846_1022 (size=12104) 2024-11-21T13:24:00,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741847_1023 (size=12104) 2024-11-21T13:24:00,318 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:00,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T13:24:00,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:00,326 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/66099bbe30d0468da4df28f9dd8a9684 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/66099bbe30d0468da4df28f9dd8a9684 2024-11-21T13:24:00,332 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/06928b2608704771bc06c9ca84fdf92f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/06928b2608704771bc06c9ca84fdf92f 2024-11-21T13:24:00,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/48d17d6424274fa5a1c7caf28e77a209 is 50, key is test_row_0/A:col10/1732195440314/Put/seqid=0 2024-11-21T13:24:00,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195500335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,361 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/A of 78998c23688e9623a40dfb78a0d6fff8 into 06928b2608704771bc06c9ca84fdf92f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:00,361 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195500342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,362 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/B of 78998c23688e9623a40dfb78a0d6fff8 into 66099bbe30d0468da4df28f9dd8a9684(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:00,362 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195500332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,362 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,372 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/A, priority=13, startTime=1732195440138; duration=0sec 2024-11-21T13:24:00,373 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:00,373 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:A 2024-11-21T13:24:00,373 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:00,382 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:00,383 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/C is initiating minor compaction (all files) 2024-11-21T13:24:00,383 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/C in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:00,383 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/c1ba87c9e4954cdb804ec2d34fb71310, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/4534e7cc58164897b98d56a4dab94737, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b402ef22736549cf9e8e04ce513a1a29] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=35.2 K 2024-11-21T13:24:00,384 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1ba87c9e4954cdb804ec2d34fb71310, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732195436415 2024-11-21T13:24:00,389 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/B, priority=13, startTime=1732195440175; duration=0sec 2024-11-21T13:24:00,389 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:00,389 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:B 2024-11-21T13:24:00,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741848_1024 (size=12001) 2024-11-21T13:24:00,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195500360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,397 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/48d17d6424274fa5a1c7caf28e77a209 2024-11-21T13:24:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195500360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,398 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,406 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4534e7cc58164897b98d56a4dab94737, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1732195436624 2024-11-21T13:24:00,407 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting b402ef22736549cf9e8e04ce513a1a29, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732195437881 2024-11-21T13:24:00,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/a2cce96de17240bcb9cf9ea362095476 is 50, key is test_row_0/B:col10/1732195440314/Put/seqid=0 2024-11-21T13:24:00,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T13:24:00,470 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-21T13:24:00,471 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#C#compaction#13 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:00,472 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T13:24:00,473 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-21T13:24:00,473 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T13:24:00,473 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-21T13:24:00,474 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-21T13:24:00,474 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e6b894daaca4404486c60cc7929e68c5 is 50, key is test_row_0/C:col10/1732195439060/Put/seqid=0 2024-11-21T13:24:00,474 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-21T13:24:00,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195500470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,484 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195500470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195500470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,487 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741849_1025 (size=12001) 2024-11-21T13:24:00,505 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/a2cce96de17240bcb9cf9ea362095476 2024-11-21T13:24:00,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195500500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195500511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/2626b67585944674921aaf7e3c1c9648 is 50, key is test_row_0/C:col10/1732195440314/Put/seqid=0 2024-11-21T13:24:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-21T13:24:00,570 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:00,571 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:00,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741850_1026 (size=12104) 2024-11-21T13:24:00,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-21T13:24:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T13:24:00,575 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:00,577 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:00,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:00,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741851_1027 (size=12001) 2024-11-21T13:24:00,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T13:24:00,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T13:24:00,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:00,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:00,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:00,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:00,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:00,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195500690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195500691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195500691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195500738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195500748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:00,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:00,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T13:24:00,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T13:24:00,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:00,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:00,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:00,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:00,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:00,991 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e6b894daaca4404486c60cc7929e68c5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e6b894daaca4404486c60cc7929e68c5 2024-11-21T13:24:01,003 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/C of 78998c23688e9623a40dfb78a0d6fff8 into e6b894daaca4404486c60cc7929e68c5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:01,003 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:01,003 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/C, priority=13, startTime=1732195440176; duration=0sec 2024-11-21T13:24:01,003 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:01,004 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:C 2024-11-21T13:24:01,037 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/2626b67585944674921aaf7e3c1c9648 2024-11-21T13:24:01,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T13:24:01,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:01,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:01,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:01,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:01,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:01,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:01,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195501050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/48d17d6424274fa5a1c7caf28e77a209 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/48d17d6424274fa5a1c7caf28e77a209 2024-11-21T13:24:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195501051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195501052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,060 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,061 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195501061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,063 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195501061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,065 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/48d17d6424274fa5a1c7caf28e77a209, entries=150, sequenceid=77, filesize=11.7 K 2024-11-21T13:24:01,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/a2cce96de17240bcb9cf9ea362095476 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a2cce96de17240bcb9cf9ea362095476 2024-11-21T13:24:01,084 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a2cce96de17240bcb9cf9ea362095476, entries=150, sequenceid=77, filesize=11.7 K 2024-11-21T13:24:01,086 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/2626b67585944674921aaf7e3c1c9648 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/2626b67585944674921aaf7e3c1c9648 2024-11-21T13:24:01,112 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/2626b67585944674921aaf7e3c1c9648, entries=150, sequenceid=77, filesize=11.7 K 2024-11-21T13:24:01,115 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 78998c23688e9623a40dfb78a0d6fff8 in 797ms, sequenceid=77, compaction requested=false 2024-11-21T13:24:01,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:01,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T13:24:01,211 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-21T13:24:01,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:01,213 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T13:24:01,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/8aa9fe2d4303426c92f8fcfe9d36d542 is 50, key is test_row_0/A:col10/1732195440330/Put/seqid=0 2024-11-21T13:24:01,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741852_1028 (size=12001) 2024-11-21T13:24:01,266 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/8aa9fe2d4303426c92f8fcfe9d36d542 2024-11-21T13:24:01,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/f48246852438427fb3a483c89237e725 is 50, key is test_row_0/B:col10/1732195440330/Put/seqid=0 2024-11-21T13:24:01,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741853_1029 (size=12001) 2024-11-21T13:24:01,365 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/f48246852438427fb3a483c89237e725 2024-11-21T13:24:01,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/917066414d6d4e21ae22f4df89809ab5 is 50, key is test_row_0/C:col10/1732195440330/Put/seqid=0 2024-11-21T13:24:01,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741854_1030 (size=12001) 2024-11-21T13:24:01,466 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/917066414d6d4e21ae22f4df89809ab5 2024-11-21T13:24:01,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/8aa9fe2d4303426c92f8fcfe9d36d542 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/8aa9fe2d4303426c92f8fcfe9d36d542 2024-11-21T13:24:01,504 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/8aa9fe2d4303426c92f8fcfe9d36d542, entries=150, sequenceid=90, filesize=11.7 K 2024-11-21T13:24:01,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/f48246852438427fb3a483c89237e725 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/f48246852438427fb3a483c89237e725 2024-11-21T13:24:01,524 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/f48246852438427fb3a483c89237e725, entries=150, sequenceid=90, filesize=11.7 K 2024-11-21T13:24:01,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/917066414d6d4e21ae22f4df89809ab5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/917066414d6d4e21ae22f4df89809ab5 2024-11-21T13:24:01,539 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/917066414d6d4e21ae22f4df89809ab5, entries=150, sequenceid=90, filesize=11.7 K 2024-11-21T13:24:01,540 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 78998c23688e9623a40dfb78a0d6fff8 in 329ms, sequenceid=90, compaction requested=true 2024-11-21T13:24:01,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:01,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:01,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-21T13:24:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-21T13:24:01,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-21T13:24:01,551 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 967 msec 2024-11-21T13:24:01,556 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 982 msec 2024-11-21T13:24:01,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:01,612 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:01,613 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:24:01,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/b59e2652aac749ddbafbf3494c5bcbb1 is 50, key is test_row_0/A:col10/1732195441610/Put/seqid=0 2024-11-21T13:24:01,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741855_1031 (size=12001) 2024-11-21T13:24:01,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/b59e2652aac749ddbafbf3494c5bcbb1 2024-11-21T13:24:01,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/7a9baffe5d7145f09da9e7a5ad54e447 is 50, key is test_row_0/B:col10/1732195441610/Put/seqid=0 2024-11-21T13:24:01,699 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195501682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195501686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195501696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195501700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,708 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195501701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,709 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-21T13:24:01,711 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:01,712 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:01,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-21T13:24:01,718 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:01,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T13:24:01,719 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:01,719 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:01,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741856_1032 (size=12001) 2024-11-21T13:24:01,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/7a9baffe5d7145f09da9e7a5ad54e447 2024-11-21T13:24:01,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e30cbc7e37e7432b96e58eba821ee084 is 50, key is test_row_0/C:col10/1732195441610/Put/seqid=0 2024-11-21T13:24:01,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741857_1033 (size=12001) 2024-11-21T13:24:01,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e30cbc7e37e7432b96e58eba821ee084 2024-11-21T13:24:01,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/b59e2652aac749ddbafbf3494c5bcbb1 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/b59e2652aac749ddbafbf3494c5bcbb1 2024-11-21T13:24:01,816 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195501810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,817 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195501810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195501810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195501819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195501819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/b59e2652aac749ddbafbf3494c5bcbb1, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T13:24:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:01,827 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:01,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T13:24:01,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/7a9baffe5d7145f09da9e7a5ad54e447 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/7a9baffe5d7145f09da9e7a5ad54e447 2024-11-21T13:24:01,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/7a9baffe5d7145f09da9e7a5ad54e447, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T13:24:01,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e30cbc7e37e7432b96e58eba821ee084 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e30cbc7e37e7432b96e58eba821ee084 2024-11-21T13:24:01,866 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e30cbc7e37e7432b96e58eba821ee084, entries=150, sequenceid=103, filesize=11.7 K 2024-11-21T13:24:01,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 78998c23688e9623a40dfb78a0d6fff8 in 256ms, sequenceid=103, compaction requested=true 2024-11-21T13:24:01,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:01,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:01,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:01,869 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:01,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:01,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:01,869 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:01,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:01,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:01,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-21T13:24:01,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:01,874 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T13:24:01,878 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:01,878 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/A is initiating minor compaction (all files) 2024-11-21T13:24:01,878 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/A in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:01,878 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/06928b2608704771bc06c9ca84fdf92f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/48d17d6424274fa5a1c7caf28e77a209, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/8aa9fe2d4303426c92f8fcfe9d36d542, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/b59e2652aac749ddbafbf3494c5bcbb1] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=47.0 K 2024-11-21T13:24:01,880 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06928b2608704771bc06c9ca84fdf92f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732195437881 2024-11-21T13:24:01,882 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:01,882 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/B is initiating minor compaction (all files) 2024-11-21T13:24:01,882 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/B in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:01,882 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/66099bbe30d0468da4df28f9dd8a9684, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a2cce96de17240bcb9cf9ea362095476, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/f48246852438427fb3a483c89237e725, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/7a9baffe5d7145f09da9e7a5ad54e447] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=47.0 K 2024-11-21T13:24:01,883 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48d17d6424274fa5a1c7caf28e77a209, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732195439130 2024-11-21T13:24:01,884 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8aa9fe2d4303426c92f8fcfe9d36d542, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732195440330 2024-11-21T13:24:01,884 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 66099bbe30d0468da4df28f9dd8a9684, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732195437881 2024-11-21T13:24:01,885 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting a2cce96de17240bcb9cf9ea362095476, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732195439130 2024-11-21T13:24:01,886 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting b59e2652aac749ddbafbf3494c5bcbb1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732195441598 2024-11-21T13:24:01,887 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f48246852438427fb3a483c89237e725, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732195440330 2024-11-21T13:24:01,889 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a9baffe5d7145f09da9e7a5ad54e447, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732195441598 2024-11-21T13:24:01,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/615001c099d940cb98bbfe9c534f4110 is 50, key is test_row_0/A:col10/1732195441678/Put/seqid=0 2024-11-21T13:24:01,930 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#A#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:01,936 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#B#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:01,936 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/da7154480a04485b990e8c00f5c7f25b is 50, key is test_row_0/A:col10/1732195441610/Put/seqid=0 2024-11-21T13:24:01,938 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/d554dc7694264b149e236115326de517 is 50, key is test_row_0/B:col10/1732195441610/Put/seqid=0 2024-11-21T13:24:01,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741858_1034 (size=12001) 2024-11-21T13:24:01,952 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/615001c099d940cb98bbfe9c534f4110 2024-11-21T13:24:01,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741859_1035 (size=12241) 2024-11-21T13:24:01,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/a119cd4822a84561940fbbbf66d6aee3 is 50, key is test_row_0/B:col10/1732195441678/Put/seqid=0 2024-11-21T13:24:01,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741860_1036 (size=12241) 2024-11-21T13:24:02,004 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/da7154480a04485b990e8c00f5c7f25b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/da7154480a04485b990e8c00f5c7f25b 2024-11-21T13:24:02,013 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/d554dc7694264b149e236115326de517 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/d554dc7694264b149e236115326de517 2024-11-21T13:24:02,023 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/A of 78998c23688e9623a40dfb78a0d6fff8 into da7154480a04485b990e8c00f5c7f25b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:02,023 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:02,023 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/A, priority=12, startTime=1732195441869; duration=0sec 2024-11-21T13:24:02,023 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:02,023 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:A 2024-11-21T13:24:02,023 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:02,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:02,028 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/B of 78998c23688e9623a40dfb78a0d6fff8 into d554dc7694264b149e236115326de517(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:02,028 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:02,028 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/B, priority=12, startTime=1732195441869; duration=0sec 2024-11-21T13:24:02,029 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:02,029 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:B 2024-11-21T13:24:02,031 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:02,031 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/C is initiating minor compaction (all files) 2024-11-21T13:24:02,031 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/C in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:02,031 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e6b894daaca4404486c60cc7929e68c5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/2626b67585944674921aaf7e3c1c9648, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/917066414d6d4e21ae22f4df89809ab5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e30cbc7e37e7432b96e58eba821ee084] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=47.0 K 2024-11-21T13:24:02,032 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6b894daaca4404486c60cc7929e68c5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732195437881 2024-11-21T13:24:02,033 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2626b67585944674921aaf7e3c1c9648, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732195439130 2024-11-21T13:24:02,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741861_1037 (size=12001) 2024-11-21T13:24:02,034 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 917066414d6d4e21ae22f4df89809ab5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732195440330 2024-11-21T13:24:02,035 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting e30cbc7e37e7432b96e58eba821ee084, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732195441598 2024-11-21T13:24:02,035 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/a119cd4822a84561940fbbbf66d6aee3 2024-11-21T13:24:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T13:24:02,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/64148b4a052c4daa877ed9276f329451 is 50, key is test_row_0/C:col10/1732195441678/Put/seqid=0 2024-11-21T13:24:02,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195502060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,070 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,071 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195502062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,075 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195502064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,075 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195502070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195502070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,078 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/1b4ee870741c433b808621995918aa89 is 50, key is test_row_0/C:col10/1732195441610/Put/seqid=0 2024-11-21T13:24:02,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741862_1038 (size=12001) 2024-11-21T13:24:02,138 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/64148b4a052c4daa877ed9276f329451 2024-11-21T13:24:02,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741863_1039 (size=12241) 2024-11-21T13:24:02,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/615001c099d940cb98bbfe9c534f4110 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/615001c099d940cb98bbfe9c534f4110 2024-11-21T13:24:02,167 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/1b4ee870741c433b808621995918aa89 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/1b4ee870741c433b808621995918aa89 2024-11-21T13:24:02,171 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/615001c099d940cb98bbfe9c534f4110, entries=150, sequenceid=127, filesize=11.7 K 2024-11-21T13:24:02,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/a119cd4822a84561940fbbbf66d6aee3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a119cd4822a84561940fbbbf66d6aee3 2024-11-21T13:24:02,181 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/C of 78998c23688e9623a40dfb78a0d6fff8 into 1b4ee870741c433b808621995918aa89(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:02,181 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:02,181 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/C, priority=12, startTime=1732195441869; duration=0sec 2024-11-21T13:24:02,182 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:02,182 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:C 2024-11-21T13:24:02,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195502179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,185 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195502179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,186 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195502179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,193 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a119cd4822a84561940fbbbf66d6aee3, entries=150, sequenceid=127, filesize=11.7 K 2024-11-21T13:24:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195502189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195502190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/64148b4a052c4daa877ed9276f329451 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/64148b4a052c4daa877ed9276f329451 2024-11-21T13:24:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,198 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,206 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/64148b4a052c4daa877ed9276f329451, entries=150, sequenceid=127, filesize=11.7 K 2024-11-21T13:24:02,209 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 78998c23688e9623a40dfb78a0d6fff8 in 335ms, sequenceid=127, compaction requested=false 2024-11-21T13:24:02,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:02,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:02,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-21T13:24:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-21T13:24:02,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-21T13:24:02,221 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 492 msec 2024-11-21T13:24:02,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 511 msec 2024-11-21T13:24:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-21T13:24:02,349 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:02,350 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-21T13:24:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T13:24:02,357 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:02,358 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:02,359 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:02,393 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:02,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T13:24:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:02,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/678f34b5e444437388953730b8ef47d9 is 50, key is test_row_0/A:col10/1732195442391/Put/seqid=0 2024-11-21T13:24:02,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741864_1040 (size=14541) 2024-11-21T13:24:02,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/678f34b5e444437388953730b8ef47d9 2024-11-21T13:24:02,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195502435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195502432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/7ae9c9e8b8704a77aa4a55f150eeb145 is 50, key is test_row_0/B:col10/1732195442391/Put/seqid=0 2024-11-21T13:24:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,451 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195502432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195502451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195502452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,455 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T13:24:02,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741865_1041 (size=12151) 2024-11-21T13:24:02,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/7ae9c9e8b8704a77aa4a55f150eeb145 2024-11-21T13:24:02,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/71ba5538ced74be6b64342202b3e00b0 is 50, key is test_row_0/C:col10/1732195442391/Put/seqid=0 2024-11-21T13:24:02,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T13:24:02,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:02,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:02,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:02,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:02,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:02,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741866_1042 (size=12151) 2024-11-21T13:24:02,525 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/71ba5538ced74be6b64342202b3e00b0 2024-11-21T13:24:02,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/678f34b5e444437388953730b8ef47d9 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/678f34b5e444437388953730b8ef47d9 2024-11-21T13:24:02,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/678f34b5e444437388953730b8ef47d9, entries=200, sequenceid=145, filesize=14.2 K 2024-11-21T13:24:02,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/7ae9c9e8b8704a77aa4a55f150eeb145 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/7ae9c9e8b8704a77aa4a55f150eeb145 2024-11-21T13:24:02,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195502560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/7ae9c9e8b8704a77aa4a55f150eeb145, entries=150, sequenceid=145, filesize=11.9 K 2024-11-21T13:24:02,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/71ba5538ced74be6b64342202b3e00b0 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/71ba5538ced74be6b64342202b3e00b0 2024-11-21T13:24:02,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195502560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195502561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195502561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195502561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,568 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/71ba5538ced74be6b64342202b3e00b0, entries=150, sequenceid=145, filesize=11.9 K 2024-11-21T13:24:02,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 78998c23688e9623a40dfb78a0d6fff8 in 185ms, sequenceid=145, compaction requested=true 2024-11-21T13:24:02,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:02,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:02,579 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:02,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:02,579 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:02,581 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:02,581 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:02,581 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/A is initiating minor compaction (all files) 2024-11-21T13:24:02,581 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/B is initiating minor compaction (all files) 2024-11-21T13:24:02,582 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/A in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:02,582 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/B in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:02,582 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/da7154480a04485b990e8c00f5c7f25b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/615001c099d940cb98bbfe9c534f4110, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/678f34b5e444437388953730b8ef47d9] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=37.9 K 2024-11-21T13:24:02,582 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/d554dc7694264b149e236115326de517, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a119cd4822a84561940fbbbf66d6aee3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/7ae9c9e8b8704a77aa4a55f150eeb145] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=35.5 K 2024-11-21T13:24:02,582 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting da7154480a04485b990e8c00f5c7f25b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732195441598 2024-11-21T13:24:02,582 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting d554dc7694264b149e236115326de517, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732195441598 2024-11-21T13:24:02,583 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 615001c099d940cb98bbfe9c534f4110, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732195441676 2024-11-21T13:24:02,584 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting a119cd4822a84561940fbbbf66d6aee3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732195441676 2024-11-21T13:24:02,582 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:02,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:02,585 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 678f34b5e444437388953730b8ef47d9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732195442060 2024-11-21T13:24:02,585 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ae9c9e8b8704a77aa4a55f150eeb145, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732195442063 2024-11-21T13:24:02,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:02,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:02,604 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:02,605 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/6eb563398bfa49ccad161e048014d103 is 50, key is test_row_0/A:col10/1732195442391/Put/seqid=0 2024-11-21T13:24:02,611 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#B#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:02,611 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/c60015aee08749e299d85908239fb45a is 50, key is test_row_0/B:col10/1732195442391/Put/seqid=0 2024-11-21T13:24:02,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741868_1044 (size=12493) 2024-11-21T13:24:02,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741867_1043 (size=12493) 2024-11-21T13:24:02,639 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/c60015aee08749e299d85908239fb45a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/c60015aee08749e299d85908239fb45a 2024-11-21T13:24:02,651 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/B of 78998c23688e9623a40dfb78a0d6fff8 into c60015aee08749e299d85908239fb45a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:02,651 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:02,651 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/B, priority=13, startTime=1732195442579; duration=0sec 2024-11-21T13:24:02,651 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:02,651 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:B 2024-11-21T13:24:02,651 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:02,653 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:02,653 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/C is initiating minor compaction (all files) 2024-11-21T13:24:02,653 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/C in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:02,654 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/1b4ee870741c433b808621995918aa89, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/64148b4a052c4daa877ed9276f329451, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/71ba5538ced74be6b64342202b3e00b0] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=35.5 K 2024-11-21T13:24:02,655 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b4ee870741c433b808621995918aa89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1732195441598 2024-11-21T13:24:02,655 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 64148b4a052c4daa877ed9276f329451, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732195441676 2024-11-21T13:24:02,656 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 71ba5538ced74be6b64342202b3e00b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732195442063 2024-11-21T13:24:02,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T13:24:02,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-21T13:24:02,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:02,671 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T13:24:02,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/79f78da77f034c5b9c68cbdf28e9a77c is 50, key is test_row_0/A:col10/1732195442441/Put/seqid=0 2024-11-21T13:24:02,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741869_1045 (size=12151) 2024-11-21T13:24:02,702 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#C#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:02,703 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/dd0bc0358fdf491a8a5940b439356574 is 50, key is test_row_0/C:col10/1732195442391/Put/seqid=0 2024-11-21T13:24:02,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741870_1046 (size=12493) 2024-11-21T13:24:02,732 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/dd0bc0358fdf491a8a5940b439356574 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/dd0bc0358fdf491a8a5940b439356574 2024-11-21T13:24:02,746 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/C of 78998c23688e9623a40dfb78a0d6fff8 into dd0bc0358fdf491a8a5940b439356574(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:02,746 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:02,746 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/C, priority=13, startTime=1732195442584; duration=0sec 2024-11-21T13:24:02,746 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:02,746 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:C 2024-11-21T13:24:02,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:02,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:02,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195502795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195502795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,806 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195502800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195502803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195502803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,809 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195502909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,912 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195502909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,913 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195502919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195502919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:02,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195502920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:02,925 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:02,926 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:02,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T13:24:03,048 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/6eb563398bfa49ccad161e048014d103 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/6eb563398bfa49ccad161e048014d103 2024-11-21T13:24:03,060 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/A of 78998c23688e9623a40dfb78a0d6fff8 into 6eb563398bfa49ccad161e048014d103(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:03,061 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:03,061 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/A, priority=13, startTime=1732195442579; duration=0sec 2024-11-21T13:24:03,061 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:03,062 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:A 2024-11-21T13:24:03,100 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/79f78da77f034c5b9c68cbdf28e9a77c 2024-11-21T13:24:03,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195503120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:03,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195503120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,124 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:03,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/62f2809bb366453d9f56765a765e752f is 50, key is test_row_0/B:col10/1732195442441/Put/seqid=0 2024-11-21T13:24:03,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:03,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195503129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:03,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:03,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:03,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195503130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:03,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195503130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:03,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:03,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:03,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741871_1047 (size=12151) 2024-11-21T13:24:03,157 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/62f2809bb366453d9f56765a765e752f 2024-11-21T13:24:03,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/34aeef11d584439ab2458465869d9270 is 50, key is test_row_0/C:col10/1732195442441/Put/seqid=0 2024-11-21T13:24:03,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741872_1048 (size=12151) 2024-11-21T13:24:03,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195503430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:03,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195503430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,434 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:03,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:03,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195503439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:03,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:03,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195503439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:03,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:03,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195503439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,443 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:03,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,445 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:03,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T13:24:03,603 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/34aeef11d584439ab2458465869d9270 2024-11-21T13:24:03,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/79f78da77f034c5b9c68cbdf28e9a77c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/79f78da77f034c5b9c68cbdf28e9a77c 2024-11-21T13:24:03,620 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/79f78da77f034c5b9c68cbdf28e9a77c, entries=150, sequenceid=167, filesize=11.9 K 2024-11-21T13:24:03,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/62f2809bb366453d9f56765a765e752f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/62f2809bb366453d9f56765a765e752f 2024-11-21T13:24:03,637 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/62f2809bb366453d9f56765a765e752f, entries=150, sequenceid=167, filesize=11.9 K 2024-11-21T13:24:03,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/34aeef11d584439ab2458465869d9270 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/34aeef11d584439ab2458465869d9270 2024-11-21T13:24:03,653 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/34aeef11d584439ab2458465869d9270, entries=150, sequenceid=167, filesize=11.9 K 2024-11-21T13:24:03,657 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 78998c23688e9623a40dfb78a0d6fff8 in 985ms, sequenceid=167, compaction requested=false 2024-11-21T13:24:03,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:03,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:03,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-21T13:24:03,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-21T13:24:03,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-21T13:24:03,665 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3020 sec 2024-11-21T13:24:03,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.3230 sec 2024-11-21T13:24:03,958 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:03,958 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T13:24:03,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:03,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/06b30ec2da774465b14be3930ae05ed3 is 50, key is test_row_0/A:col10/1732195443957/Put/seqid=0 2024-11-21T13:24:03,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741873_1049 (size=12151) 2024-11-21T13:24:03,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/06b30ec2da774465b14be3930ae05ed3 2024-11-21T13:24:04,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/00c71c8aa2984f8ea9e87a77d9e6a87a is 50, key is test_row_0/B:col10/1732195443957/Put/seqid=0 2024-11-21T13:24:04,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741874_1050 (size=12151) 2024-11-21T13:24:04,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/00c71c8aa2984f8ea9e87a77d9e6a87a 2024-11-21T13:24:04,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/9c1c789b2caf458bb71c01b8e9fd19c2 is 50, key is test_row_0/C:col10/1732195443957/Put/seqid=0 2024-11-21T13:24:04,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741875_1051 (size=12151) 2024-11-21T13:24:04,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/9c1c789b2caf458bb71c01b8e9fd19c2 2024-11-21T13:24:04,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/06b30ec2da774465b14be3930ae05ed3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/06b30ec2da774465b14be3930ae05ed3 2024-11-21T13:24:04,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/06b30ec2da774465b14be3930ae05ed3, entries=150, sequenceid=187, filesize=11.9 K 2024-11-21T13:24:04,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/00c71c8aa2984f8ea9e87a77d9e6a87a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/00c71c8aa2984f8ea9e87a77d9e6a87a 2024-11-21T13:24:04,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195504106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/00c71c8aa2984f8ea9e87a77d9e6a87a, entries=150, sequenceid=187, filesize=11.9 K 2024-11-21T13:24:04,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/9c1c789b2caf458bb71c01b8e9fd19c2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/9c1c789b2caf458bb71c01b8e9fd19c2 2024-11-21T13:24:04,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195504114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195504116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195504117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195504131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,136 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/9c1c789b2caf458bb71c01b8e9fd19c2, entries=150, sequenceid=187, filesize=11.9 K 2024-11-21T13:24:04,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=107.34 KB/109920 for 78998c23688e9623a40dfb78a0d6fff8 in 184ms, sequenceid=187, compaction requested=true 2024-11-21T13:24:04,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:04,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:04,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:04,143 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:04,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:04,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:04,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:04,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:04,143 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:04,146 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:04,146 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:04,147 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/A is initiating minor compaction (all files) 2024-11-21T13:24:04,147 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/B is initiating minor compaction (all files) 2024-11-21T13:24:04,147 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/B in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:04,147 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/A in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:04,147 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/c60015aee08749e299d85908239fb45a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/62f2809bb366453d9f56765a765e752f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/00c71c8aa2984f8ea9e87a77d9e6a87a] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=35.9 K 2024-11-21T13:24:04,147 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/6eb563398bfa49ccad161e048014d103, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/79f78da77f034c5b9c68cbdf28e9a77c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/06b30ec2da774465b14be3930ae05ed3] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=35.9 K 2024-11-21T13:24:04,148 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting c60015aee08749e299d85908239fb45a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732195442063 2024-11-21T13:24:04,148 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6eb563398bfa49ccad161e048014d103, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732195442063 2024-11-21T13:24:04,148 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79f78da77f034c5b9c68cbdf28e9a77c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732195442432 2024-11-21T13:24:04,149 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 62f2809bb366453d9f56765a765e752f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732195442432 2024-11-21T13:24:04,150 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 00c71c8aa2984f8ea9e87a77d9e6a87a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732195442800 2024-11-21T13:24:04,150 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06b30ec2da774465b14be3930ae05ed3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732195442800 2024-11-21T13:24:04,174 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#A#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:04,177 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/5010acd118b345f5835738ade9a0ab14 is 50, key is test_row_0/A:col10/1732195443957/Put/seqid=0 2024-11-21T13:24:04,180 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:04,180 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/1e56050c07e54a55adeafa1957a47254 is 50, key is test_row_0/B:col10/1732195443957/Put/seqid=0 2024-11-21T13:24:04,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741876_1052 (size=12595) 2024-11-21T13:24:04,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:04,234 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T13:24:04,245 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/5010acd118b345f5835738ade9a0ab14 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/5010acd118b345f5835738ade9a0ab14 2024-11-21T13:24:04,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741877_1053 (size=12595) 2024-11-21T13:24:04,257 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/A of 78998c23688e9623a40dfb78a0d6fff8 into 5010acd118b345f5835738ade9a0ab14(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:04,257 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:04,257 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/A, priority=13, startTime=1732195444143; duration=0sec 2024-11-21T13:24:04,257 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:04,257 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:A 2024-11-21T13:24:04,257 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:04,259 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/1e56050c07e54a55adeafa1957a47254 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/1e56050c07e54a55adeafa1957a47254 2024-11-21T13:24:04,260 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:04,260 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/C is initiating minor compaction (all files) 2024-11-21T13:24:04,260 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/C in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:04,260 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/dd0bc0358fdf491a8a5940b439356574, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/34aeef11d584439ab2458465869d9270, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/9c1c789b2caf458bb71c01b8e9fd19c2] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=35.9 K 2024-11-21T13:24:04,261 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd0bc0358fdf491a8a5940b439356574, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1732195442063 2024-11-21T13:24:04,263 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34aeef11d584439ab2458465869d9270, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732195442432 2024-11-21T13:24:04,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/48cf8bc4d8074c49abc6b23a62adc739 is 50, key is test_row_0/A:col10/1732195444112/Put/seqid=0 2024-11-21T13:24:04,266 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c1c789b2caf458bb71c01b8e9fd19c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732195442800 2024-11-21T13:24:04,274 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/B of 78998c23688e9623a40dfb78a0d6fff8 into 1e56050c07e54a55adeafa1957a47254(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:04,274 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:04,274 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/B, priority=13, startTime=1732195444143; duration=0sec 2024-11-21T13:24:04,276 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:04,276 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:B 2024-11-21T13:24:04,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195504271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,286 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#C#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:04,286 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5e2186c9ca0048b699ef02f87441d0d2 is 50, key is test_row_0/C:col10/1732195443957/Put/seqid=0 2024-11-21T13:24:04,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741878_1054 (size=12151) 2024-11-21T13:24:04,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195504280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,296 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195504283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,297 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195504283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195504284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741879_1055 (size=12595) 2024-11-21T13:24:04,330 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5e2186c9ca0048b699ef02f87441d0d2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5e2186c9ca0048b699ef02f87441d0d2 2024-11-21T13:24:04,342 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/C of 78998c23688e9623a40dfb78a0d6fff8 into 5e2186c9ca0048b699ef02f87441d0d2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:04,342 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:04,343 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/C, priority=13, startTime=1732195444143; duration=0sec 2024-11-21T13:24:04,343 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:04,343 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:C 2024-11-21T13:24:04,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195504389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195504400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195504400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195504409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,412 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195504409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,413 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-21T13:24:04,499 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:04,500 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:04,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-21T13:24:04,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T13:24:04,506 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:04,509 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:04,509 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:04,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T13:24:04,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195504609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195504610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195504610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195504619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195504619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T13:24:04,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:04,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:04,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:04,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:04,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:04,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:04,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/48cf8bc4d8074c49abc6b23a62adc739 2024-11-21T13:24:04,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/dba46caa194c4d8397d56eb7802974ed is 50, key is test_row_0/B:col10/1732195444112/Put/seqid=0 2024-11-21T13:24:04,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741880_1056 (size=12151) 2024-11-21T13:24:04,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/dba46caa194c4d8397d56eb7802974ed 2024-11-21T13:24:04,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/07590f08a1254ee6b788d9d212dfe19d is 50, key is test_row_0/C:col10/1732195444112/Put/seqid=0 2024-11-21T13:24:04,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741881_1057 (size=12151) 2024-11-21T13:24:04,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T13:24:04,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T13:24:04,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:04,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:04,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:04,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:04,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:04,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:04,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195504919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195504920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,925 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195504920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,928 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,929 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,933 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:04,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195504930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195504930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:04,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T13:24:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:04,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:04,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:04,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:04,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:05,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T13:24:05,136 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T13:24:05,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:05,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:05,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:05,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:05,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:05,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/07590f08a1254ee6b788d9d212dfe19d 2024-11-21T13:24:05,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/48cf8bc4d8074c49abc6b23a62adc739 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/48cf8bc4d8074c49abc6b23a62adc739 2024-11-21T13:24:05,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/48cf8bc4d8074c49abc6b23a62adc739, entries=150, sequenceid=207, filesize=11.9 K 2024-11-21T13:24:05,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/dba46caa194c4d8397d56eb7802974ed as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/dba46caa194c4d8397d56eb7802974ed 2024-11-21T13:24:05,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/dba46caa194c4d8397d56eb7802974ed, entries=150, sequenceid=207, filesize=11.9 K 2024-11-21T13:24:05,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/07590f08a1254ee6b788d9d212dfe19d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/07590f08a1254ee6b788d9d212dfe19d 2024-11-21T13:24:05,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/07590f08a1254ee6b788d9d212dfe19d, entries=150, sequenceid=207, filesize=11.9 K 2024-11-21T13:24:05,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=100.63 KB/103050 for 78998c23688e9623a40dfb78a0d6fff8 in 992ms, sequenceid=207, compaction requested=false 2024-11-21T13:24:05,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:05,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-21T13:24:05,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:05,292 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T13:24:05,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/744a7dccd7ba44a9aae646d6bcca9af7 is 50, key is test_row_0/A:col10/1732195444280/Put/seqid=0 2024-11-21T13:24:05,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741882_1058 (size=12151) 2024-11-21T13:24:05,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:05,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:05,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195505456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195505456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195505461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,466 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195505463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,470 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195505461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195505571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,573 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195505571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195505571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,574 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195505579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195505579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T13:24:05,747 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/744a7dccd7ba44a9aae646d6bcca9af7 2024-11-21T13:24:05,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/a9cbc2d29f1e41cd8800e38a1e78e644 is 50, key is test_row_0/B:col10/1732195444280/Put/seqid=0 2024-11-21T13:24:05,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195505789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195505789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195505789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195505789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:05,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195505789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:05,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:05,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741883_1059 (size=12151) 2024-11-21T13:24:06,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195506100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:06,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:06,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195506100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:06,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,101 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:06,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:06,104 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:06,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195506102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:06,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,105 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:06,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:06,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195506110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:06,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195506110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:06,202 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/a9cbc2d29f1e41cd8800e38a1e78e644 2024-11-21T13:24:06,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5d9df3b9411e4809a45efa3c62cdf9ff is 50, key is test_row_0/C:col10/1732195444280/Put/seqid=0 2024-11-21T13:24:06,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741884_1060 (size=12151) 2024-11-21T13:24:06,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:06,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:06,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195506609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195506610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:06,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195506610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,613 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:06,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:06,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195506619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:06,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195506619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:06,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:06,647 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5d9df3b9411e4809a45efa3c62cdf9ff 2024-11-21T13:24:06,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T13:24:06,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/744a7dccd7ba44a9aae646d6bcca9af7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/744a7dccd7ba44a9aae646d6bcca9af7 2024-11-21T13:24:06,659 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/744a7dccd7ba44a9aae646d6bcca9af7, entries=150, sequenceid=228, filesize=11.9 K 2024-11-21T13:24:06,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/a9cbc2d29f1e41cd8800e38a1e78e644 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a9cbc2d29f1e41cd8800e38a1e78e644 2024-11-21T13:24:06,666 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a9cbc2d29f1e41cd8800e38a1e78e644, entries=150, sequenceid=228, filesize=11.9 K 2024-11-21T13:24:06,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5d9df3b9411e4809a45efa3c62cdf9ff as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5d9df3b9411e4809a45efa3c62cdf9ff 2024-11-21T13:24:06,674 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5d9df3b9411e4809a45efa3c62cdf9ff, entries=150, sequenceid=228, filesize=11.9 K 2024-11-21T13:24:06,677 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 78998c23688e9623a40dfb78a0d6fff8 in 1386ms, sequenceid=228, compaction requested=true 2024-11-21T13:24:06,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:06,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:06,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-21T13:24:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-21T13:24:06,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-21T13:24:06,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1750 sec 2024-11-21T13:24:06,691 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.1890 sec 2024-11-21T13:24:07,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:07,623 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:07,623 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T13:24:07,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/28781f68c92f419ab0074195ed2658e2 is 50, key is test_row_0/A:col10/1732195445451/Put/seqid=0 2024-11-21T13:24:07,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741885_1061 (size=12151) 2024-11-21T13:24:07,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195507642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195507641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195507643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195507647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195507647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195507749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,761 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195507759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195507759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195507759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195507759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195507959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195507969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195507969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195507970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:07,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:07,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195507970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:07,972 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/28781f68c92f419ab0074195ed2658e2 2024-11-21T13:24:08,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/aa0f37a38c2a4cbda0d87eae3163d308 is 50, key is test_row_0/B:col10/1732195445451/Put/seqid=0 2024-11-21T13:24:08,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741886_1062 (size=12151) 2024-11-21T13:24:08,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/aa0f37a38c2a4cbda0d87eae3163d308 2024-11-21T13:24:08,069 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5283ae31a9664e95a8fbb00fa1dab45e is 50, key is test_row_0/C:col10/1732195445451/Put/seqid=0 2024-11-21T13:24:08,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741887_1063 (size=12151) 2024-11-21T13:24:08,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195508269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195508280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195508280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,283 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195508280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195508280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,284 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5283ae31a9664e95a8fbb00fa1dab45e 2024-11-21T13:24:08,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/28781f68c92f419ab0074195ed2658e2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/28781f68c92f419ab0074195ed2658e2 2024-11-21T13:24:08,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/28781f68c92f419ab0074195ed2658e2, entries=150, sequenceid=247, filesize=11.9 K 2024-11-21T13:24:08,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/aa0f37a38c2a4cbda0d87eae3163d308 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/aa0f37a38c2a4cbda0d87eae3163d308 2024-11-21T13:24:08,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/aa0f37a38c2a4cbda0d87eae3163d308, entries=150, sequenceid=247, filesize=11.9 K 2024-11-21T13:24:08,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5283ae31a9664e95a8fbb00fa1dab45e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5283ae31a9664e95a8fbb00fa1dab45e 2024-11-21T13:24:08,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5283ae31a9664e95a8fbb00fa1dab45e, entries=150, sequenceid=247, filesize=11.9 K 2024-11-21T13:24:08,513 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 78998c23688e9623a40dfb78a0d6fff8 in 890ms, sequenceid=247, compaction requested=true 2024-11-21T13:24:08,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:08,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:08,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:08,514 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:08,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:08,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:08,514 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:08,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:08,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:08,515 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:08,515 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/B is initiating minor compaction (all files) 2024-11-21T13:24:08,515 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/B in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:08,516 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/1e56050c07e54a55adeafa1957a47254, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/dba46caa194c4d8397d56eb7802974ed, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a9cbc2d29f1e41cd8800e38a1e78e644, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/aa0f37a38c2a4cbda0d87eae3163d308] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=47.9 K 2024-11-21T13:24:08,516 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:08,516 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/A is initiating minor compaction (all files) 2024-11-21T13:24:08,516 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/A in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:08,516 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e56050c07e54a55adeafa1957a47254, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732195442800 2024-11-21T13:24:08,516 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/5010acd118b345f5835738ade9a0ab14, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/48cf8bc4d8074c49abc6b23a62adc739, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/744a7dccd7ba44a9aae646d6bcca9af7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/28781f68c92f419ab0074195ed2658e2] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=47.9 K 2024-11-21T13:24:08,517 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting dba46caa194c4d8397d56eb7802974ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732195444112 2024-11-21T13:24:08,517 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5010acd118b345f5835738ade9a0ab14, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732195442800 2024-11-21T13:24:08,517 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting a9cbc2d29f1e41cd8800e38a1e78e644, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732195444266 2024-11-21T13:24:08,517 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48cf8bc4d8074c49abc6b23a62adc739, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732195444112 2024-11-21T13:24:08,518 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 744a7dccd7ba44a9aae646d6bcca9af7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732195444266 2024-11-21T13:24:08,518 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting aa0f37a38c2a4cbda0d87eae3163d308, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732195445451 2024-11-21T13:24:08,518 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28781f68c92f419ab0074195ed2658e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732195445451 2024-11-21T13:24:08,529 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#A#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:08,529 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:08,530 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/879471d8b9c1497ab6b30ef0edd8c59f is 50, key is test_row_0/B:col10/1732195445451/Put/seqid=0 2024-11-21T13:24:08,530 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/1f4fa24a3c024ea88ec824c65e33ec4a is 50, key is test_row_0/A:col10/1732195445451/Put/seqid=0 2024-11-21T13:24:08,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741888_1064 (size=12731) 2024-11-21T13:24:08,551 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/1f4fa24a3c024ea88ec824c65e33ec4a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/1f4fa24a3c024ea88ec824c65e33ec4a 2024-11-21T13:24:08,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741889_1065 (size=12731) 2024-11-21T13:24:08,561 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/A of 78998c23688e9623a40dfb78a0d6fff8 into 1f4fa24a3c024ea88ec824c65e33ec4a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:08,563 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:08,563 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/A, priority=12, startTime=1732195448513; duration=0sec 2024-11-21T13:24:08,563 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:08,563 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:A 2024-11-21T13:24:08,563 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:08,565 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:08,565 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/C is initiating minor compaction (all files) 2024-11-21T13:24:08,565 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/C in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:08,565 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5e2186c9ca0048b699ef02f87441d0d2, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/07590f08a1254ee6b788d9d212dfe19d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5d9df3b9411e4809a45efa3c62cdf9ff, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5283ae31a9664e95a8fbb00fa1dab45e] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=47.9 K 2024-11-21T13:24:08,565 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e2186c9ca0048b699ef02f87441d0d2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1732195442800 2024-11-21T13:24:08,566 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07590f08a1254ee6b788d9d212dfe19d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732195444112 2024-11-21T13:24:08,566 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d9df3b9411e4809a45efa3c62cdf9ff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732195444266 2024-11-21T13:24:08,567 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5283ae31a9664e95a8fbb00fa1dab45e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732195445451 2024-11-21T13:24:08,579 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#C#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:08,580 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/64451b08721d404f99a97852bb42fb59 is 50, key is test_row_0/C:col10/1732195445451/Put/seqid=0 2024-11-21T13:24:08,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741890_1066 (size=12731) 2024-11-21T13:24:08,602 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/64451b08721d404f99a97852bb42fb59 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/64451b08721d404f99a97852bb42fb59 2024-11-21T13:24:08,610 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/C of 78998c23688e9623a40dfb78a0d6fff8 into 64451b08721d404f99a97852bb42fb59(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:08,610 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:08,610 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/C, priority=12, startTime=1732195448514; duration=0sec 2024-11-21T13:24:08,610 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:08,610 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:C 2024-11-21T13:24:08,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-21T13:24:08,670 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:08,671 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-21T13:24:08,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T13:24:08,675 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:08,676 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:08,676 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T13:24:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:08,784 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:08,784 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T13:24:08,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/f1b69f15e7c94c6eaed2acfb60ab8632 is 50, key is test_row_0/A:col10/1732195447644/Put/seqid=0 2024-11-21T13:24:08,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741891_1067 (size=12251) 2024-11-21T13:24:08,814 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195508811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195508813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195508813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195508813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195508814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T13:24:08,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:08,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:08,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:08,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:08,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:08,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:08,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195508924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,926 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195508929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195508929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,931 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195508930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195508930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:08,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:08,969 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/879471d8b9c1497ab6b30ef0edd8c59f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/879471d8b9c1497ab6b30ef0edd8c59f 2024-11-21T13:24:08,978 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/B of 78998c23688e9623a40dfb78a0d6fff8 into 879471d8b9c1497ab6b30ef0edd8c59f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:08,978 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:08,978 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/B, priority=12, startTime=1732195448514; duration=0sec 2024-11-21T13:24:08,978 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:08,978 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:B 2024-11-21T13:24:08,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T13:24:08,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:08,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:08,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:08,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:08,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:08,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T13:24:09,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195509129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,132 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T13:24:09,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:09,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:09,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195509139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:09,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195509139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195509139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,143 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195509140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/f1b69f15e7c94c6eaed2acfb60ab8632 2024-11-21T13:24:09,209 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/0953fea064944a83b5682cdd5292acdd is 50, key is test_row_0/B:col10/1732195447644/Put/seqid=0 2024-11-21T13:24:09,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741892_1068 (size=12251) 2024-11-21T13:24:09,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T13:24:09,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:09,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:09,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:09,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T13:24:09,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195509439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T13:24:09,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:09,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:09,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:09,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195509451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195509451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195509459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195509459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T13:24:09,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:09,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:09,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:09,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/0953fea064944a83b5682cdd5292acdd 2024-11-21T13:24:09,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/ffe1bb9a9095444e8e7cd2a6cd6a1ae0 is 50, key is test_row_0/C:col10/1732195447644/Put/seqid=0 2024-11-21T13:24:09,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741893_1069 (size=12251) 2024-11-21T13:24:09,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T13:24:09,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:09,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:09,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:09,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T13:24:09,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T13:24:09,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:09,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:09,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:09,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:09,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195509949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195509959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195509959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195509969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195509969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:10,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T13:24:10,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:10,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:10,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:10,063 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:10,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:10,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:10,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/ffe1bb9a9095444e8e7cd2a6cd6a1ae0 2024-11-21T13:24:10,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/f1b69f15e7c94c6eaed2acfb60ab8632 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/f1b69f15e7c94c6eaed2acfb60ab8632 2024-11-21T13:24:10,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/f1b69f15e7c94c6eaed2acfb60ab8632, entries=150, sequenceid=268, filesize=12.0 K 2024-11-21T13:24:10,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/0953fea064944a83b5682cdd5292acdd as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0953fea064944a83b5682cdd5292acdd 2024-11-21T13:24:10,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0953fea064944a83b5682cdd5292acdd, entries=150, sequenceid=268, filesize=12.0 K 2024-11-21T13:24:10,109 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/ffe1bb9a9095444e8e7cd2a6cd6a1ae0 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/ffe1bb9a9095444e8e7cd2a6cd6a1ae0 2024-11-21T13:24:10,116 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/ffe1bb9a9095444e8e7cd2a6cd6a1ae0, entries=150, sequenceid=268, filesize=12.0 K 2024-11-21T13:24:10,118 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 78998c23688e9623a40dfb78a0d6fff8 in 1333ms, sequenceid=268, compaction requested=false 2024-11-21T13:24:10,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:10,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-21T13:24:10,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:10,236 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T13:24:10,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/ed0d0ceb639f4133b172f199fe6886bf is 50, key is test_row_0/A:col10/1732195448807/Put/seqid=0 2024-11-21T13:24:10,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741894_1070 (size=12301) 2024-11-21T13:24:10,261 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/ed0d0ceb639f4133b172f199fe6886bf 2024-11-21T13:24:10,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/e9c7b4a80ddf49358ae4f26f32751d01 is 50, key is test_row_0/B:col10/1732195448807/Put/seqid=0 2024-11-21T13:24:10,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741895_1071 (size=12301) 2024-11-21T13:24:10,679 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/e9c7b4a80ddf49358ae4f26f32751d01 2024-11-21T13:24:10,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/3a53bd2b147d4998bd0ab8507766da6d is 50, key is test_row_0/C:col10/1732195448807/Put/seqid=0 2024-11-21T13:24:10,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741896_1072 (size=12301) 2024-11-21T13:24:10,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T13:24:10,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:10,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:11,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,046 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195510999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195510999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195511047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195511048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195511055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,098 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/3a53bd2b147d4998bd0ab8507766da6d 2024-11-21T13:24:11,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/ed0d0ceb639f4133b172f199fe6886bf as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ed0d0ceb639f4133b172f199fe6886bf 2024-11-21T13:24:11,120 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ed0d0ceb639f4133b172f199fe6886bf, entries=150, sequenceid=286, filesize=12.0 K 2024-11-21T13:24:11,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/e9c7b4a80ddf49358ae4f26f32751d01 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/e9c7b4a80ddf49358ae4f26f32751d01 2024-11-21T13:24:11,133 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/e9c7b4a80ddf49358ae4f26f32751d01, entries=150, sequenceid=286, filesize=12.0 K 2024-11-21T13:24:11,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/3a53bd2b147d4998bd0ab8507766da6d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/3a53bd2b147d4998bd0ab8507766da6d 2024-11-21T13:24:11,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195511159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195511160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195511160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195511160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195511160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,168 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,172 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,173 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/3a53bd2b147d4998bd0ab8507766da6d, entries=150, sequenceid=286, filesize=12.0 K 2024-11-21T13:24:11,174 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 78998c23688e9623a40dfb78a0d6fff8 in 939ms, sequenceid=286, compaction requested=true 2024-11-21T13:24:11,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:11,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:11,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-21T13:24:11,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-21T13:24:11,182 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-21T13:24:11,183 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5000 sec 2024-11-21T13:24:11,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.5130 sec 2024-11-21T13:24:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:11,374 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:11,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T13:24:11,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/2d6d98afaeb94fd881cd70afddf10d9c is 50, key is test_row_0/A:col10/1732195451371/Put/seqid=0 2024-11-21T13:24:11,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195511404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195511406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,411 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195511408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195511408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,419 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195511409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,420 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741897_1073 (size=14741) 2024-11-21T13:24:11,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/2d6d98afaeb94fd881cd70afddf10d9c 2024-11-21T13:24:11,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/1df39bebbf27407286aa6386e18ba945 is 50, key is test_row_0/B:col10/1732195451371/Put/seqid=0 2024-11-21T13:24:11,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741898_1074 (size=12301) 2024-11-21T13:24:11,479 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/1df39bebbf27407286aa6386e18ba945 2024-11-21T13:24:11,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e9d7560e846a4cefa10fa4b7731304f6 is 50, key is test_row_0/C:col10/1732195451371/Put/seqid=0 2024-11-21T13:24:11,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741899_1075 (size=12301) 2024-11-21T13:24:11,513 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=308 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e9d7560e846a4cefa10fa4b7731304f6 2024-11-21T13:24:11,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/2d6d98afaeb94fd881cd70afddf10d9c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/2d6d98afaeb94fd881cd70afddf10d9c 2024-11-21T13:24:11,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195511520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,525 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195511520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195511530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195511530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195511530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/2d6d98afaeb94fd881cd70afddf10d9c, entries=200, sequenceid=308, filesize=14.4 K 2024-11-21T13:24:11,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/1df39bebbf27407286aa6386e18ba945 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/1df39bebbf27407286aa6386e18ba945 2024-11-21T13:24:11,545 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/1df39bebbf27407286aa6386e18ba945, entries=150, sequenceid=308, filesize=12.0 K 2024-11-21T13:24:11,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e9d7560e846a4cefa10fa4b7731304f6 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e9d7560e846a4cefa10fa4b7731304f6 2024-11-21T13:24:11,562 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e9d7560e846a4cefa10fa4b7731304f6, entries=150, sequenceid=308, filesize=12.0 K 2024-11-21T13:24:11,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 78998c23688e9623a40dfb78a0d6fff8 in 190ms, sequenceid=308, compaction requested=true 2024-11-21T13:24:11,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:11,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:11,565 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:11,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:11,565 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:11,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:11,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:11,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:11,565 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:11,566 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:11,566 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/A is initiating minor compaction (all files) 2024-11-21T13:24:11,566 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/A in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:11,567 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/1f4fa24a3c024ea88ec824c65e33ec4a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/f1b69f15e7c94c6eaed2acfb60ab8632, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ed0d0ceb639f4133b172f199fe6886bf, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/2d6d98afaeb94fd881cd70afddf10d9c] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=50.8 K 2024-11-21T13:24:11,567 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f4fa24a3c024ea88ec824c65e33ec4a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732195445451 2024-11-21T13:24:11,570 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49584 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:11,570 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/B is initiating minor compaction (all files) 2024-11-21T13:24:11,570 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/B in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:11,570 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/879471d8b9c1497ab6b30ef0edd8c59f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0953fea064944a83b5682cdd5292acdd, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/e9c7b4a80ddf49358ae4f26f32751d01, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/1df39bebbf27407286aa6386e18ba945] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=48.4 K 2024-11-21T13:24:11,571 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1b69f15e7c94c6eaed2acfb60ab8632, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732195447630 2024-11-21T13:24:11,571 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed0d0ceb639f4133b172f199fe6886bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732195448807 2024-11-21T13:24:11,571 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 879471d8b9c1497ab6b30ef0edd8c59f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732195445451 2024-11-21T13:24:11,572 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d6d98afaeb94fd881cd70afddf10d9c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732195451002 2024-11-21T13:24:11,576 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 0953fea064944a83b5682cdd5292acdd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732195447630 2024-11-21T13:24:11,576 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e9c7b4a80ddf49358ae4f26f32751d01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732195448807 2024-11-21T13:24:11,577 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 1df39bebbf27407286aa6386e18ba945, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732195451002 2024-11-21T13:24:11,588 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#A#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:11,589 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/38f94365e95a47a8ae1790eb7442fc30 is 50, key is test_row_0/A:col10/1732195451371/Put/seqid=0 2024-11-21T13:24:11,589 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#B#compaction#64 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:11,591 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/cf088bad836a45b49dd96b92f169e1ee is 50, key is test_row_0/B:col10/1732195451371/Put/seqid=0 2024-11-21T13:24:11,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741900_1076 (size=13017) 2024-11-21T13:24:11,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741901_1077 (size=13017) 2024-11-21T13:24:11,606 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/cf088bad836a45b49dd96b92f169e1ee as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/cf088bad836a45b49dd96b92f169e1ee 2024-11-21T13:24:11,630 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/B of 78998c23688e9623a40dfb78a0d6fff8 into cf088bad836a45b49dd96b92f169e1ee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:11,630 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:11,630 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/B, priority=12, startTime=1732195451565; duration=0sec 2024-11-21T13:24:11,631 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:11,631 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:B 2024-11-21T13:24:11,631 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:11,634 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49584 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:11,634 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/C is initiating minor compaction (all files) 2024-11-21T13:24:11,634 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/C in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:11,634 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/64451b08721d404f99a97852bb42fb59, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/ffe1bb9a9095444e8e7cd2a6cd6a1ae0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/3a53bd2b147d4998bd0ab8507766da6d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e9d7560e846a4cefa10fa4b7731304f6] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=48.4 K 2024-11-21T13:24:11,635 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 64451b08721d404f99a97852bb42fb59, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1732195445451 2024-11-21T13:24:11,635 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting ffe1bb9a9095444e8e7cd2a6cd6a1ae0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1732195447630 2024-11-21T13:24:11,636 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a53bd2b147d4998bd0ab8507766da6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1732195448807 2024-11-21T13:24:11,638 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e9d7560e846a4cefa10fa4b7731304f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732195451002 2024-11-21T13:24:11,686 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#C#compaction#65 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:11,687 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/791ac545bd804eaf9d5420653c6635c4 is 50, key is test_row_0/C:col10/1732195451371/Put/seqid=0 2024-11-21T13:24:11,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:11,732 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T13:24:11,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741902_1078 (size=13017) 2024-11-21T13:24:11,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/c6b7a13da5a04ae5a681cd8338fed9c8 is 50, key is test_row_0/A:col10/1732195451392/Put/seqid=0 2024-11-21T13:24:11,750 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/791ac545bd804eaf9d5420653c6635c4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/791ac545bd804eaf9d5420653c6635c4 2024-11-21T13:24:11,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741903_1079 (size=14741) 2024-11-21T13:24:11,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/c6b7a13da5a04ae5a681cd8338fed9c8 2024-11-21T13:24:11,797 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/C of 78998c23688e9623a40dfb78a0d6fff8 into 791ac545bd804eaf9d5420653c6635c4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:11,797 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:11,797 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/C, priority=12, startTime=1732195451565; duration=0sec 2024-11-21T13:24:11,798 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:11,798 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:C 2024-11-21T13:24:11,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195511822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/393c24602f4b4b6e8bc6df2cfd00b35e is 50, key is test_row_0/B:col10/1732195451392/Put/seqid=0 2024-11-21T13:24:11,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195511827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,840 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195511829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195511828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195511831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,848 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,849 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741904_1080 (size=12301) 2024-11-21T13:24:11,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/393c24602f4b4b6e8bc6df2cfd00b35e 2024-11-21T13:24:11,887 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e470acee75704e018fd17501f289a101 is 50, key is test_row_0/C:col10/1732195451392/Put/seqid=0 2024-11-21T13:24:11,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741905_1081 (size=12301) 2024-11-21T13:24:11,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e470acee75704e018fd17501f289a101 2024-11-21T13:24:11,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/c6b7a13da5a04ae5a681cd8338fed9c8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/c6b7a13da5a04ae5a681cd8338fed9c8 2024-11-21T13:24:11,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/c6b7a13da5a04ae5a681cd8338fed9c8, entries=200, sequenceid=324, filesize=14.4 K 2024-11-21T13:24:11,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/393c24602f4b4b6e8bc6df2cfd00b35e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/393c24602f4b4b6e8bc6df2cfd00b35e 2024-11-21T13:24:11,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/393c24602f4b4b6e8bc6df2cfd00b35e, entries=150, sequenceid=324, filesize=12.0 K 2024-11-21T13:24:11,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e470acee75704e018fd17501f289a101 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e470acee75704e018fd17501f289a101 2024-11-21T13:24:11,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:11,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195511939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:11,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:11,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e470acee75704e018fd17501f289a101, entries=150, sequenceid=324, filesize=12.0 K 2024-11-21T13:24:11,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 78998c23688e9623a40dfb78a0d6fff8 in 212ms, sequenceid=324, compaction requested=false 2024-11-21T13:24:11,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:11,953 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:11,953 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T13:24:11,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/ac956bf3d278468daef3095b687c2afe is 50, key is test_row_0/A:col10/1732195451826/Put/seqid=0 2024-11-21T13:24:11,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741906_1082 (size=12301) 2024-11-21T13:24:12,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195511991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195511991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,003 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195511993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195512001, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,013 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/38f94365e95a47a8ae1790eb7442fc30 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/38f94365e95a47a8ae1790eb7442fc30 2024-11-21T13:24:12,029 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/A of 78998c23688e9623a40dfb78a0d6fff8 into 38f94365e95a47a8ae1790eb7442fc30(size=12.7 K), total size for store is 27.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:12,029 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:12,029 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/A, priority=12, startTime=1732195451564; duration=0sec 2024-11-21T13:24:12,029 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:12,029 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:A 2024-11-21T13:24:12,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195512109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195512109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195512109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,121 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195512109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195512149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195512319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195512329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195512329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195512329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,332 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/ac956bf3d278468daef3095b687c2afe 2024-11-21T13:24:12,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/3783f1b3592c4d5ea696807297b7a7ac is 50, key is test_row_0/B:col10/1732195451826/Put/seqid=0 2024-11-21T13:24:12,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741907_1083 (size=12301) 2024-11-21T13:24:12,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195512460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195512629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195512639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195512639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195512640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/3783f1b3592c4d5ea696807297b7a7ac 2024-11-21T13:24:12,800 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5a63764cf56a434fa2af97f8c6d2efa0 is 50, key is test_row_0/C:col10/1732195451826/Put/seqid=0 2024-11-21T13:24:12,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741908_1084 (size=12301) 2024-11-21T13:24:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-21T13:24:12,839 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:12,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-21T13:24:12,842 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T13:24:12,844 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:12,844 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:12,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T13:24:12,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:12,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195512969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:12,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T13:24:12,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:12,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:12,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:12,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:12,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:12,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:13,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:13,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195513138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:13,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:13,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:13,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:13,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195513149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:13,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T13:24:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:13,150 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:13,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:13,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195513149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:13,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:13,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:13,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:13,151 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:13,151 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:13,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:13,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:13,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T13:24:13,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:13,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195513159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:13,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:13,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:13,161 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:13,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5a63764cf56a434fa2af97f8c6d2efa0 2024-11-21T13:24:13,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/ac956bf3d278468daef3095b687c2afe as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ac956bf3d278468daef3095b687c2afe 2024-11-21T13:24:13,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ac956bf3d278468daef3095b687c2afe, entries=150, sequenceid=350, filesize=12.0 K 2024-11-21T13:24:13,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/3783f1b3592c4d5ea696807297b7a7ac as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/3783f1b3592c4d5ea696807297b7a7ac 2024-11-21T13:24:13,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/3783f1b3592c4d5ea696807297b7a7ac, entries=150, sequenceid=350, filesize=12.0 K 2024-11-21T13:24:13,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5a63764cf56a434fa2af97f8c6d2efa0 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5a63764cf56a434fa2af97f8c6d2efa0 2024-11-21T13:24:13,251 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5a63764cf56a434fa2af97f8c6d2efa0, entries=150, sequenceid=350, filesize=12.0 K 2024-11-21T13:24:13,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 78998c23688e9623a40dfb78a0d6fff8 in 1301ms, sequenceid=350, compaction requested=true 2024-11-21T13:24:13,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:13,254 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:13,255 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:13,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:13,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:13,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:13,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:13,255 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T13:24:13,255 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:13,256 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:13,256 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:13,256 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/A is initiating minor compaction (all files) 2024-11-21T13:24:13,256 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/B is initiating minor compaction (all files) 2024-11-21T13:24:13,257 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/A in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:13,257 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/B in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:13,257 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/38f94365e95a47a8ae1790eb7442fc30, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/c6b7a13da5a04ae5a681cd8338fed9c8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ac956bf3d278468daef3095b687c2afe] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=39.1 K 2024-11-21T13:24:13,257 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/cf088bad836a45b49dd96b92f169e1ee, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/393c24602f4b4b6e8bc6df2cfd00b35e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/3783f1b3592c4d5ea696807297b7a7ac] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=36.7 K 2024-11-21T13:24:13,257 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting cf088bad836a45b49dd96b92f169e1ee, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732195451002 2024-11-21T13:24:13,257 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38f94365e95a47a8ae1790eb7442fc30, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732195451002 2024-11-21T13:24:13,258 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6b7a13da5a04ae5a681cd8338fed9c8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732195451392 2024-11-21T13:24:13,258 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 393c24602f4b4b6e8bc6df2cfd00b35e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732195451392 2024-11-21T13:24:13,258 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac956bf3d278468daef3095b687c2afe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732195451808 2024-11-21T13:24:13,259 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 3783f1b3592c4d5ea696807297b7a7ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732195451808 2024-11-21T13:24:13,286 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#B#compaction#72 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:13,286 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/0638e4f4547848cbbcc1691b66400112 is 50, key is test_row_0/B:col10/1732195451826/Put/seqid=0 2024-11-21T13:24:13,297 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#A#compaction#73 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:13,297 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/eefcd5af46c341a78684b6dc9e9284e1 is 50, key is test_row_0/A:col10/1732195451826/Put/seqid=0 2024-11-21T13:24:13,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-21T13:24:13,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:13,304 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:24:13,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/fe9968c9eb0c46ca86cdc46762de491d is 50, key is test_row_0/A:col10/1732195451997/Put/seqid=0 2024-11-21T13:24:13,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741909_1085 (size=13119) 2024-11-21T13:24:13,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741910_1086 (size=13119) 2024-11-21T13:24:13,335 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/0638e4f4547848cbbcc1691b66400112 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0638e4f4547848cbbcc1691b66400112 2024-11-21T13:24:13,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741911_1087 (size=12301) 2024-11-21T13:24:13,342 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/eefcd5af46c341a78684b6dc9e9284e1 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/eefcd5af46c341a78684b6dc9e9284e1 2024-11-21T13:24:13,342 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/fe9968c9eb0c46ca86cdc46762de491d 2024-11-21T13:24:13,346 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/B of 78998c23688e9623a40dfb78a0d6fff8 into 0638e4f4547848cbbcc1691b66400112(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:13,346 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:13,346 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/B, priority=13, startTime=1732195453255; duration=0sec 2024-11-21T13:24:13,347 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:13,347 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:B 2024-11-21T13:24:13,348 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:13,349 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:13,350 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/C is initiating minor compaction (all files) 2024-11-21T13:24:13,350 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/C in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:13,350 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/791ac545bd804eaf9d5420653c6635c4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e470acee75704e018fd17501f289a101, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5a63764cf56a434fa2af97f8c6d2efa0] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=36.7 K 2024-11-21T13:24:13,352 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 791ac545bd804eaf9d5420653c6635c4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=308, earliestPutTs=1732195451002 2024-11-21T13:24:13,352 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e470acee75704e018fd17501f289a101, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732195451392 2024-11-21T13:24:13,353 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a63764cf56a434fa2af97f8c6d2efa0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732195451808 2024-11-21T13:24:13,355 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/A of 78998c23688e9623a40dfb78a0d6fff8 into eefcd5af46c341a78684b6dc9e9284e1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:13,355 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:13,355 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/A, priority=13, startTime=1732195453254; duration=0sec 2024-11-21T13:24:13,355 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:13,355 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:A 2024-11-21T13:24:13,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/bfbc94a543394eb39d3abe3625e56ac8 is 50, key is test_row_0/B:col10/1732195451997/Put/seqid=0 2024-11-21T13:24:13,378 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#C#compaction#76 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:13,378 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/4b8ad7bb75d148c9a1dfca6ae5e23cfd is 50, key is test_row_0/C:col10/1732195451826/Put/seqid=0 2024-11-21T13:24:13,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741912_1088 (size=12301) 2024-11-21T13:24:13,392 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/bfbc94a543394eb39d3abe3625e56ac8 2024-11-21T13:24:13,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741913_1089 (size=13119) 2024-11-21T13:24:13,403 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/4b8ad7bb75d148c9a1dfca6ae5e23cfd as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/4b8ad7bb75d148c9a1dfca6ae5e23cfd 2024-11-21T13:24:13,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/b9c40090232e43688cced95d9beee657 is 50, key is test_row_0/C:col10/1732195451997/Put/seqid=0 2024-11-21T13:24:13,421 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/C of 78998c23688e9623a40dfb78a0d6fff8 into 4b8ad7bb75d148c9a1dfca6ae5e23cfd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:13,421 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:13,421 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/C, priority=13, startTime=1732195453255; duration=0sec 2024-11-21T13:24:13,421 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:13,421 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:C 2024-11-21T13:24:13,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741914_1090 (size=12301) 2024-11-21T13:24:13,425 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/b9c40090232e43688cced95d9beee657 2024-11-21T13:24:13,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/fe9968c9eb0c46ca86cdc46762de491d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/fe9968c9eb0c46ca86cdc46762de491d 2024-11-21T13:24:13,444 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/fe9968c9eb0c46ca86cdc46762de491d, entries=150, sequenceid=362, filesize=12.0 K 2024-11-21T13:24:13,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/bfbc94a543394eb39d3abe3625e56ac8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bfbc94a543394eb39d3abe3625e56ac8 2024-11-21T13:24:13,454 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bfbc94a543394eb39d3abe3625e56ac8, entries=150, sequenceid=362, filesize=12.0 K 2024-11-21T13:24:13,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/b9c40090232e43688cced95d9beee657 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b9c40090232e43688cced95d9beee657 2024-11-21T13:24:13,463 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b9c40090232e43688cced95d9beee657, entries=150, sequenceid=362, filesize=12.0 K 2024-11-21T13:24:13,464 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 78998c23688e9623a40dfb78a0d6fff8 in 160ms, sequenceid=362, compaction requested=false 2024-11-21T13:24:13,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:13,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:13,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-21T13:24:13,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-21T13:24:13,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-21T13:24:13,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 622 msec 2024-11-21T13:24:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T13:24:13,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 630 msec 2024-11-21T13:24:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-21T13:24:13,979 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:13,980 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-21T13:24:13,984 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:13,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T13:24:13,986 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:13,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:14,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:14,046 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:14,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:24:14,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/a2d4a7a1251f44049535e6f467189b3b is 50, key is test_row_0/A:col10/1732195454046/Put/seqid=0 2024-11-21T13:24:14,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741915_1091 (size=14741) 2024-11-21T13:24:14,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/a2d4a7a1251f44049535e6f467189b3b 2024-11-21T13:24:14,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T13:24:14,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/cc1151fc93024f2fbc7aeab8a19565ec is 50, key is test_row_0/B:col10/1732195454046/Put/seqid=0 2024-11-21T13:24:14,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T13:24:14,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:14,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:14,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:14,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:14,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:14,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:14,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741916_1092 (size=12301) 2024-11-21T13:24:14,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/cc1151fc93024f2fbc7aeab8a19565ec 2024-11-21T13:24:14,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195514160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:14,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:14,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195514160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:14,167 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 78998c23688e9623a40dfb78a0d6fff8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:14,168 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_1' in 78998c23688e9623a40dfb78a0d6fff8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:14,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195514163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:14,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,171 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:14,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:14,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195514164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:14,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/b188b095a15a4b4bb4b9ee144a5cc699 is 50, key is test_row_0/C:col10/1732195454046/Put/seqid=0 2024-11-21T13:24:14,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:14,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195514170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:14,174 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 78998c23688e9623a40dfb78a0d6fff8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,175 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:14,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741917_1093 (size=12301) 2024-11-21T13:24:14,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:14,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195514289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195514289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:14,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T13:24:14,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:14,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:14,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:14,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:14,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:14,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T13:24:14,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T13:24:14,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:14,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:14,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:14,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:14,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:14,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:14,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:14,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195514500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:14,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195514500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:14,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T13:24:14,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:14,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:14,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:14,605 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:14,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:14,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T13:24:14,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:14,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=377 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/b188b095a15a4b4bb4b9ee144a5cc699 2024-11-21T13:24:14,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/a2d4a7a1251f44049535e6f467189b3b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/a2d4a7a1251f44049535e6f467189b3b 2024-11-21T13:24:14,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/a2d4a7a1251f44049535e6f467189b3b, entries=200, sequenceid=377, filesize=14.4 K 2024-11-21T13:24:14,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/cc1151fc93024f2fbc7aeab8a19565ec as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/cc1151fc93024f2fbc7aeab8a19565ec 2024-11-21T13:24:14,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/cc1151fc93024f2fbc7aeab8a19565ec, entries=150, sequenceid=377, filesize=12.0 K 2024-11-21T13:24:14,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/b188b095a15a4b4bb4b9ee144a5cc699 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b188b095a15a4b4bb4b9ee144a5cc699 2024-11-21T13:24:14,663 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b188b095a15a4b4bb4b9ee144a5cc699, entries=150, sequenceid=377, filesize=12.0 K 2024-11-21T13:24:14,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 78998c23688e9623a40dfb78a0d6fff8 in 619ms, sequenceid=377, compaction requested=true 2024-11-21T13:24:14,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:14,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:14,666 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:14,666 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:14,666 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:14,668 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:14,668 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/A is initiating minor compaction (all files) 2024-11-21T13:24:14,668 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/A in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:14,669 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/eefcd5af46c341a78684b6dc9e9284e1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/fe9968c9eb0c46ca86cdc46762de491d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/a2d4a7a1251f44049535e6f467189b3b] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=39.2 K 2024-11-21T13:24:14,670 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:14,670 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/B is initiating minor compaction (all files) 2024-11-21T13:24:14,670 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/B in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:14,670 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0638e4f4547848cbbcc1691b66400112, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bfbc94a543394eb39d3abe3625e56ac8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/cc1151fc93024f2fbc7aeab8a19565ec] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=36.8 K 2024-11-21T13:24:14,671 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting eefcd5af46c341a78684b6dc9e9284e1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732195451808 2024-11-21T13:24:14,672 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe9968c9eb0c46ca86cdc46762de491d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732195451957 2024-11-21T13:24:14,673 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 0638e4f4547848cbbcc1691b66400112, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732195451808 2024-11-21T13:24:14,673 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2d4a7a1251f44049535e6f467189b3b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732195454016 2024-11-21T13:24:14,673 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting bfbc94a543394eb39d3abe3625e56ac8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732195451957 2024-11-21T13:24:14,674 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting cc1151fc93024f2fbc7aeab8a19565ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732195454016 2024-11-21T13:24:14,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:14,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:14,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:14,689 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:14,698 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#A#compaction#81 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:14,699 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/acf3fefc11c44367a169e4b8be4b685d is 50, key is test_row_0/A:col10/1732195454046/Put/seqid=0 2024-11-21T13:24:14,704 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#B#compaction#82 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:14,704 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/af83a242b4804ab298a73c73a659e83a is 50, key is test_row_0/B:col10/1732195454046/Put/seqid=0 2024-11-21T13:24:14,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741918_1094 (size=13221) 2024-11-21T13:24:14,726 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/acf3fefc11c44367a169e4b8be4b685d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/acf3fefc11c44367a169e4b8be4b685d 2024-11-21T13:24:14,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741919_1095 (size=13221) 2024-11-21T13:24:14,752 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/A of 78998c23688e9623a40dfb78a0d6fff8 into acf3fefc11c44367a169e4b8be4b685d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:14,753 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:14,753 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/A, priority=13, startTime=1732195454666; duration=0sec 2024-11-21T13:24:14,753 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:14,753 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:A 2024-11-21T13:24:14,753 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:14,757 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/af83a242b4804ab298a73c73a659e83a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/af83a242b4804ab298a73c73a659e83a 2024-11-21T13:24:14,758 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:14,758 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/C is initiating minor compaction (all files) 2024-11-21T13:24:14,758 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/C in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:14,758 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/4b8ad7bb75d148c9a1dfca6ae5e23cfd, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b9c40090232e43688cced95d9beee657, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b188b095a15a4b4bb4b9ee144a5cc699] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=36.8 K 2024-11-21T13:24:14,761 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b8ad7bb75d148c9a1dfca6ae5e23cfd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732195451808 2024-11-21T13:24:14,762 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9c40090232e43688cced95d9beee657, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=362, earliestPutTs=1732195451957 2024-11-21T13:24:14,763 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting b188b095a15a4b4bb4b9ee144a5cc699, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732195454016 2024-11-21T13:24:14,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-21T13:24:14,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:14,765 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T13:24:14,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/dee4f6a9954544778b2f93623119b5bb is 50, key is test_row_0/A:col10/1732195454151/Put/seqid=0 2024-11-21T13:24:14,782 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/B of 78998c23688e9623a40dfb78a0d6fff8 into af83a242b4804ab298a73c73a659e83a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:14,782 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:14,782 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/B, priority=13, startTime=1732195454666; duration=0sec 2024-11-21T13:24:14,782 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:14,782 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:B 2024-11-21T13:24:14,785 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#C#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:14,786 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/bd262f7434db4176a9a44b165a9faa3d is 50, key is test_row_0/C:col10/1732195454046/Put/seqid=0 2024-11-21T13:24:14,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741921_1097 (size=13221) 2024-11-21T13:24:14,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741920_1096 (size=12301) 2024-11-21T13:24:14,796 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/dee4f6a9954544778b2f93623119b5bb 2024-11-21T13:24:14,804 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/bd262f7434db4176a9a44b165a9faa3d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/bd262f7434db4176a9a44b165a9faa3d 2024-11-21T13:24:14,815 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:14,815 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/C of 78998c23688e9623a40dfb78a0d6fff8 into bd262f7434db4176a9a44b165a9faa3d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:14,815 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:14,815 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/C, priority=13, startTime=1732195454689; duration=0sec 2024-11-21T13:24:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:14,816 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:14,816 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:C 2024-11-21T13:24:14,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/02b5d8b44d7b446ba6b2b923635fa10f is 50, key is test_row_0/B:col10/1732195454151/Put/seqid=0 2024-11-21T13:24:14,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195514846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,850 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:14,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195514849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:14,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741922_1098 (size=12301) 2024-11-21T13:24:14,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195514959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,961 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:14,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195514960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T13:24:15,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195515169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:15,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:15,173 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195515170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:15,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:15,254 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/02b5d8b44d7b446ba6b2b923635fa10f 2024-11-21T13:24:15,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/962575d68d3e453c90e6d5bc3ca19ae7 is 50, key is test_row_0/C:col10/1732195454151/Put/seqid=0 2024-11-21T13:24:15,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741923_1099 (size=12301) 2024-11-21T13:24:15,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195515479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:15,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195515479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:15,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:15,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:15,670 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/962575d68d3e453c90e6d5bc3ca19ae7 2024-11-21T13:24:15,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/dee4f6a9954544778b2f93623119b5bb as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/dee4f6a9954544778b2f93623119b5bb 2024-11-21T13:24:15,681 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/dee4f6a9954544778b2f93623119b5bb, entries=150, sequenceid=402, filesize=12.0 K 2024-11-21T13:24:15,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/02b5d8b44d7b446ba6b2b923635fa10f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/02b5d8b44d7b446ba6b2b923635fa10f 2024-11-21T13:24:15,688 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/02b5d8b44d7b446ba6b2b923635fa10f, entries=150, sequenceid=402, filesize=12.0 K 2024-11-21T13:24:15,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/962575d68d3e453c90e6d5bc3ca19ae7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/962575d68d3e453c90e6d5bc3ca19ae7 2024-11-21T13:24:15,695 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/962575d68d3e453c90e6d5bc3ca19ae7, entries=150, sequenceid=402, filesize=12.0 K 2024-11-21T13:24:15,698 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 78998c23688e9623a40dfb78a0d6fff8 in 932ms, sequenceid=402, compaction requested=false 2024-11-21T13:24:15,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:15,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:15,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-21T13:24:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-21T13:24:15,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-21T13:24:15,701 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7130 sec 2024-11-21T13:24:15,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.7210 sec 2024-11-21T13:24:15,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:15,991 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:15,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T13:24:16,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/3066bde85d5d4ee592ce6abafa6fe7e5 is 50, key is test_row_0/A:col10/1732195454844/Put/seqid=0 2024-11-21T13:24:16,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741924_1100 (size=17181) 2024-11-21T13:24:16,010 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/3066bde85d5d4ee592ce6abafa6fe7e5 2024-11-21T13:24:16,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/a98615bc9af34aa9b1c83c4c8fc6b56f is 50, key is test_row_0/B:col10/1732195454844/Put/seqid=0 2024-11-21T13:24:16,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741925_1101 (size=12301) 2024-11-21T13:24:16,031 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/a98615bc9af34aa9b1c83c4c8fc6b56f 2024-11-21T13:24:16,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:16,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195516036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:16,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195516037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,040 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:16,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e04c82babe8e4c3eb6b2abf20a44df8b is 50, key is test_row_0/C:col10/1732195454844/Put/seqid=0 2024-11-21T13:24:16,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741926_1102 (size=12301) 2024-11-21T13:24:16,055 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e04c82babe8e4c3eb6b2abf20a44df8b 2024-11-21T13:24:16,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/3066bde85d5d4ee592ce6abafa6fe7e5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/3066bde85d5d4ee592ce6abafa6fe7e5 2024-11-21T13:24:16,068 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/3066bde85d5d4ee592ce6abafa6fe7e5, entries=250, sequenceid=418, filesize=16.8 K 2024-11-21T13:24:16,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/a98615bc9af34aa9b1c83c4c8fc6b56f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a98615bc9af34aa9b1c83c4c8fc6b56f 2024-11-21T13:24:16,079 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a98615bc9af34aa9b1c83c4c8fc6b56f, entries=150, sequenceid=418, filesize=12.0 K 2024-11-21T13:24:16,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e04c82babe8e4c3eb6b2abf20a44df8b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e04c82babe8e4c3eb6b2abf20a44df8b 2024-11-21T13:24:16,090 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e04c82babe8e4c3eb6b2abf20a44df8b, entries=150, sequenceid=418, filesize=12.0 K 2024-11-21T13:24:16,091 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 78998c23688e9623a40dfb78a0d6fff8 in 100ms, sequenceid=418, compaction requested=true 2024-11-21T13:24:16,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:16,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:16,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:16,092 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:16,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:16,092 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:16,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:16,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 78998c23688e9623a40dfb78a0d6fff8:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:16,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:16,093 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:16,093 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42703 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:16,094 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/A is initiating minor compaction (all files) 2024-11-21T13:24:16,094 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/B is initiating minor compaction (all files) 2024-11-21T13:24:16,094 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/B in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:16,094 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/A in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:16,094 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/af83a242b4804ab298a73c73a659e83a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/02b5d8b44d7b446ba6b2b923635fa10f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a98615bc9af34aa9b1c83c4c8fc6b56f] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=36.9 K 2024-11-21T13:24:16,094 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/acf3fefc11c44367a169e4b8be4b685d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/dee4f6a9954544778b2f93623119b5bb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/3066bde85d5d4ee592ce6abafa6fe7e5] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=41.7 K 2024-11-21T13:24:16,094 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting acf3fefc11c44367a169e4b8be4b685d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732195454016 2024-11-21T13:24:16,094 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting af83a242b4804ab298a73c73a659e83a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732195454016 2024-11-21T13:24:16,095 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting dee4f6a9954544778b2f93623119b5bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732195454112 2024-11-21T13:24:16,095 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 02b5d8b44d7b446ba6b2b923635fa10f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732195454112 2024-11-21T13:24:16,095 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3066bde85d5d4ee592ce6abafa6fe7e5, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732195454838 2024-11-21T13:24:16,095 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting a98615bc9af34aa9b1c83c4c8fc6b56f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732195454844 2024-11-21T13:24:16,105 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#B#compaction#90 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:16,105 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/9433ff0dd4be4b08aa5af233fdc782a2 is 50, key is test_row_0/B:col10/1732195454844/Put/seqid=0 2024-11-21T13:24:16,106 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#A#compaction#91 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:16,106 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/6369842f2cb94976b415b0bb4db7e25a is 50, key is test_row_0/A:col10/1732195454844/Put/seqid=0 2024-11-21T13:24:16,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741927_1103 (size=13323) 2024-11-21T13:24:16,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741928_1104 (size=13323) 2024-11-21T13:24:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-21T13:24:16,129 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:16,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-21T13:24:16,132 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:16,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T13:24:16,133 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:16,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:16,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:16,152 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:16,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T13:24:16,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/499e872aa5d841e38935448c05c740a8 is 50, key is test_row_0/A:col10/1732195456034/Put/seqid=0 2024-11-21T13:24:16,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741929_1105 (size=12301) 2024-11-21T13:24:16,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/499e872aa5d841e38935448c05c740a8 2024-11-21T13:24:16,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:16,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195516183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:16,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195516185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:16,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:16,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:16,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56388 deadline: 1732195516189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:16,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/0ba195a474b74907be911f52bf9386ba is 50, key is test_row_0/B:col10/1732195456034/Put/seqid=0 2024-11-21T13:24:16,192 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 78998c23688e9623a40dfb78a0d6fff8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:16,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:16,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:16,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56372 deadline: 1732195516200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:16,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:16,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56400 deadline: 1732195516200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:16,202 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_1' in 78998c23688e9623a40dfb78a0d6fff8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:16,202 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 78998c23688e9623a40dfb78a0d6fff8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:16,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741930_1106 (size=12301) 2024-11-21T13:24:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T13:24:16,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T13:24:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:16,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195516289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:16,301 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:16,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195516299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,301 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:16,417 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-21T13:24:16,417 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-21T13:24:16,417 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:16,417 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:16,419 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-21T13:24:16,419 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:16,421 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-21T13:24:16,421 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:16,421 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:16,422 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:16,422 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:16,422 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:16,422 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:16,422 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:16,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:16,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:16,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:16,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:16,422 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:16,422 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:16,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T13:24:16,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:16,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:16,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:16,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T13:24:16,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:16,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195516499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:16,510 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195516510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:16,528 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/6369842f2cb94976b415b0bb4db7e25a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/6369842f2cb94976b415b0bb4db7e25a 2024-11-21T13:24:16,535 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/9433ff0dd4be4b08aa5af233fdc782a2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/9433ff0dd4be4b08aa5af233fdc782a2 2024-11-21T13:24:16,535 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/A of 78998c23688e9623a40dfb78a0d6fff8 into 6369842f2cb94976b415b0bb4db7e25a(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:16,536 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:16,536 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/A, priority=13, startTime=1732195456092; duration=0sec 2024-11-21T13:24:16,536 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:16,536 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:A 2024-11-21T13:24:16,536 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:16,537 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:16,537 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 78998c23688e9623a40dfb78a0d6fff8/C is initiating minor compaction (all files) 2024-11-21T13:24:16,537 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 78998c23688e9623a40dfb78a0d6fff8/C in TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:16,537 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/bd262f7434db4176a9a44b165a9faa3d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/962575d68d3e453c90e6d5bc3ca19ae7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e04c82babe8e4c3eb6b2abf20a44df8b] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp, totalSize=36.9 K 2024-11-21T13:24:16,538 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd262f7434db4176a9a44b165a9faa3d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=377, earliestPutTs=1732195454016 2024-11-21T13:24:16,538 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 962575d68d3e453c90e6d5bc3ca19ae7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1732195454112 2024-11-21T13:24:16,538 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting e04c82babe8e4c3eb6b2abf20a44df8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=418, earliestPutTs=1732195454844 2024-11-21T13:24:16,541 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/B of 78998c23688e9623a40dfb78a0d6fff8 into 9433ff0dd4be4b08aa5af233fdc782a2(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:16,541 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:16,541 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/B, priority=13, startTime=1732195456092; duration=0sec 2024-11-21T13:24:16,542 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:16,542 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:B 2024-11-21T13:24:16,546 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 78998c23688e9623a40dfb78a0d6fff8#C#compaction#94 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:16,547 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/20b6fa8d4f174df09950cbba09f0929d is 50, key is test_row_0/C:col10/1732195454844/Put/seqid=0 2024-11-21T13:24:16,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741931_1107 (size=13323) 2024-11-21T13:24:16,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T13:24:16,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:16,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/0ba195a474b74907be911f52bf9386ba 2024-11-21T13:24:16,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5d3e8a58bf51466f97252a9ccedb8ce2 is 50, key is test_row_0/C:col10/1732195456034/Put/seqid=0 2024-11-21T13:24:16,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741932_1108 (size=12301) 2024-11-21T13:24:16,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T13:24:16,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:16,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:16,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:16,749 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T13:24:16,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:16,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56368 deadline: 1732195516809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:16,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:16,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56384 deadline: 1732195516820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:16,822 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:24:16,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T13:24:16,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:16,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:16,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:16,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:16,962 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/20b6fa8d4f174df09950cbba09f0929d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/20b6fa8d4f174df09950cbba09f0929d 2024-11-21T13:24:16,967 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 78998c23688e9623a40dfb78a0d6fff8/C of 78998c23688e9623a40dfb78a0d6fff8 into 20b6fa8d4f174df09950cbba09f0929d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:16,967 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:16,967 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., storeName=78998c23688e9623a40dfb78a0d6fff8/C, priority=13, startTime=1732195456092; duration=0sec 2024-11-21T13:24:16,967 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:16,967 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 78998c23688e9623a40dfb78a0d6fff8:C 2024-11-21T13:24:17,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=441 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5d3e8a58bf51466f97252a9ccedb8ce2 2024-11-21T13:24:17,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/499e872aa5d841e38935448c05c740a8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/499e872aa5d841e38935448c05c740a8 2024-11-21T13:24:17,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/499e872aa5d841e38935448c05c740a8, entries=150, sequenceid=441, filesize=12.0 K 2024-11-21T13:24:17,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/0ba195a474b74907be911f52bf9386ba as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0ba195a474b74907be911f52bf9386ba 2024-11-21T13:24:17,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0ba195a474b74907be911f52bf9386ba, entries=150, sequenceid=441, filesize=12.0 K 2024-11-21T13:24:17,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/5d3e8a58bf51466f97252a9ccedb8ce2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5d3e8a58bf51466f97252a9ccedb8ce2 2024-11-21T13:24:17,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5d3e8a58bf51466f97252a9ccedb8ce2, entries=150, sequenceid=441, filesize=12.0 K 2024-11-21T13:24:17,055 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 78998c23688e9623a40dfb78a0d6fff8 in 903ms, sequenceid=441, compaction requested=false 2024-11-21T13:24:17,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:17,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-21T13:24:17,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:17,055 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T13:24:17,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/ffe5e8080cb148f190652e064fc73343 is 50, key is test_row_0/A:col10/1732195456181/Put/seqid=0 2024-11-21T13:24:17,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741933_1109 (size=12301) 2024-11-21T13:24:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T13:24:17,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:17,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. as already flushing 2024-11-21T13:24:17,323 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-21T13:24:17,323 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:17,323 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:17,324 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:17,324 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:17,332 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-21T13:24:17,333 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:17,333 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:17,333 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:17,333 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:17,464 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/ffe5e8080cb148f190652e064fc73343 2024-11-21T13:24:17,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/f7dd767ae6a6469e9ba3834c098e0d73 is 50, key is test_row_0/B:col10/1732195456181/Put/seqid=0 2024-11-21T13:24:17,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741934_1110 (size=12301) 2024-11-21T13:24:17,886 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/f7dd767ae6a6469e9ba3834c098e0d73 2024-11-21T13:24:17,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e7922fbebc2241cb80faa98a38847373 is 50, key is test_row_0/C:col10/1732195456181/Put/seqid=0 2024-11-21T13:24:17,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741935_1111 (size=12301) 2024-11-21T13:24:18,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T13:24:18,301 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e7922fbebc2241cb80faa98a38847373 2024-11-21T13:24:18,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/ffe5e8080cb148f190652e064fc73343 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ffe5e8080cb148f190652e064fc73343 2024-11-21T13:24:18,318 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ffe5e8080cb148f190652e064fc73343, entries=150, sequenceid=457, filesize=12.0 K 2024-11-21T13:24:18,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/f7dd767ae6a6469e9ba3834c098e0d73 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/f7dd767ae6a6469e9ba3834c098e0d73 2024-11-21T13:24:18,323 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/f7dd767ae6a6469e9ba3834c098e0d73, entries=150, sequenceid=457, filesize=12.0 K 2024-11-21T13:24:18,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/e7922fbebc2241cb80faa98a38847373 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e7922fbebc2241cb80faa98a38847373 2024-11-21T13:24:18,327 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e7922fbebc2241cb80faa98a38847373, entries=150, sequenceid=457, filesize=12.0 K 2024-11-21T13:24:18,328 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=13.42 KB/13740 for 78998c23688e9623a40dfb78a0d6fff8 in 1273ms, sequenceid=457, compaction requested=true 2024-11-21T13:24:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 78998c23688e9623a40dfb78a0d6fff8: 2024-11-21T13:24:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:18,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-21T13:24:18,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-21T13:24:18,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-21T13:24:18,332 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1970 sec 2024-11-21T13:24:18,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 2.2020 sec 2024-11-21T13:24:19,229 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T13:24:20,222 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-21T13:24:20,222 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:20,222 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:20,222 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:20,222 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:20,232 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-21T13:24:20,232 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:20,233 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:20,233 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:20,233 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:20,240 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-21T13:24:20,240 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:20,241 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:20,241 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:20,241 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:20,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-21T13:24:20,290 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 76 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3774 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3681 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1702 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5103 rows 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1702 2024-11-21T13:24:20,290 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5099 rows 2024-11-21T13:24:20,291 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T13:24:20,291 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T13:24:20,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:20,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:20,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:20,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T13:24:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:20,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T13:24:20,304 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195460303"}]},"ts":"1732195460303"} 2024-11-21T13:24:20,306 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T13:24:20,306 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T13:24:20,308 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T13:24:20,312 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78998c23688e9623a40dfb78a0d6fff8, UNASSIGN}] 2024-11-21T13:24:20,312 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78998c23688e9623a40dfb78a0d6fff8, UNASSIGN 2024-11-21T13:24:20,314 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=78998c23688e9623a40dfb78a0d6fff8, regionState=CLOSING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:20,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=78998c23688e9623a40dfb78a0d6fff8, UNASSIGN because future has completed 2024-11-21T13:24:20,316 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T13:24:20,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:24:20,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T13:24:20,475 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(122): Close 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:20,476 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T13:24:20,477 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1722): Closing 78998c23688e9623a40dfb78a0d6fff8, disabling compactions & flushes 2024-11-21T13:24:20,477 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:20,477 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:20,477 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. after waiting 0 ms 2024-11-21T13:24:20,477 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:20,478 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(2902): Flushing 78998c23688e9623a40dfb78a0d6fff8 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T13:24:20,485 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/5c27e9ce3ffe4c93b741020829d819e2 is 50, key is test_row_0/A:col10/1732195460239/Put/seqid=0 2024-11-21T13:24:20,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741936_1112 (size=12301) 2024-11-21T13:24:20,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T13:24:20,893 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/5c27e9ce3ffe4c93b741020829d819e2 2024-11-21T13:24:20,909 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/bb3dd289daac443c9ef7b0ece62f1da3 is 50, key is test_row_0/B:col10/1732195460239/Put/seqid=0 2024-11-21T13:24:20,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741937_1113 (size=12301) 2024-11-21T13:24:20,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T13:24:21,315 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/bb3dd289daac443c9ef7b0ece62f1da3 2024-11-21T13:24:21,326 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/b51a7a7c44dc49a9b390fb5dfc2a252c is 50, key is test_row_0/C:col10/1732195460239/Put/seqid=0 2024-11-21T13:24:21,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741938_1114 (size=12301) 2024-11-21T13:24:21,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T13:24:21,731 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=465 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/b51a7a7c44dc49a9b390fb5dfc2a252c 2024-11-21T13:24:21,737 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/A/5c27e9ce3ffe4c93b741020829d819e2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/5c27e9ce3ffe4c93b741020829d819e2 2024-11-21T13:24:21,743 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/5c27e9ce3ffe4c93b741020829d819e2, entries=150, sequenceid=465, filesize=12.0 K 2024-11-21T13:24:21,744 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/B/bb3dd289daac443c9ef7b0ece62f1da3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bb3dd289daac443c9ef7b0ece62f1da3 2024-11-21T13:24:21,748 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bb3dd289daac443c9ef7b0ece62f1da3, entries=150, sequenceid=465, filesize=12.0 K 2024-11-21T13:24:21,749 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/.tmp/C/b51a7a7c44dc49a9b390fb5dfc2a252c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b51a7a7c44dc49a9b390fb5dfc2a252c 2024-11-21T13:24:21,753 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b51a7a7c44dc49a9b390fb5dfc2a252c, entries=150, sequenceid=465, filesize=12.0 K 2024-11-21T13:24:21,754 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 78998c23688e9623a40dfb78a0d6fff8 in 1277ms, sequenceid=465, compaction requested=true 2024-11-21T13:24:21,754 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/f0bee866bbf346ee82dee66fbec8ccee, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/e505b33da07a412b8d9f053622c5f0e3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/06928b2608704771bc06c9ca84fdf92f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/3c522fa5413243bda6f9a14a70f94e6a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/48d17d6424274fa5a1c7caf28e77a209, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/8aa9fe2d4303426c92f8fcfe9d36d542, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/da7154480a04485b990e8c00f5c7f25b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/b59e2652aac749ddbafbf3494c5bcbb1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/615001c099d940cb98bbfe9c534f4110, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/678f34b5e444437388953730b8ef47d9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/6eb563398bfa49ccad161e048014d103, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/79f78da77f034c5b9c68cbdf28e9a77c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/5010acd118b345f5835738ade9a0ab14, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/06b30ec2da774465b14be3930ae05ed3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/48cf8bc4d8074c49abc6b23a62adc739, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/744a7dccd7ba44a9aae646d6bcca9af7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/1f4fa24a3c024ea88ec824c65e33ec4a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/28781f68c92f419ab0074195ed2658e2, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/f1b69f15e7c94c6eaed2acfb60ab8632, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ed0d0ceb639f4133b172f199fe6886bf, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/2d6d98afaeb94fd881cd70afddf10d9c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/38f94365e95a47a8ae1790eb7442fc30, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/c6b7a13da5a04ae5a681cd8338fed9c8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/eefcd5af46c341a78684b6dc9e9284e1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ac956bf3d278468daef3095b687c2afe, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/fe9968c9eb0c46ca86cdc46762de491d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/a2d4a7a1251f44049535e6f467189b3b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/acf3fefc11c44367a169e4b8be4b685d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/dee4f6a9954544778b2f93623119b5bb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/3066bde85d5d4ee592ce6abafa6fe7e5] to archive 2024-11-21T13:24:21,757 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:24:21,762 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/f0bee866bbf346ee82dee66fbec8ccee to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/f0bee866bbf346ee82dee66fbec8ccee 2024-11-21T13:24:21,763 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/e505b33da07a412b8d9f053622c5f0e3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/e505b33da07a412b8d9f053622c5f0e3 2024-11-21T13:24:21,765 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/06928b2608704771bc06c9ca84fdf92f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/06928b2608704771bc06c9ca84fdf92f 2024-11-21T13:24:21,766 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/3c522fa5413243bda6f9a14a70f94e6a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/3c522fa5413243bda6f9a14a70f94e6a 2024-11-21T13:24:21,767 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/48d17d6424274fa5a1c7caf28e77a209 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/48d17d6424274fa5a1c7caf28e77a209 2024-11-21T13:24:21,769 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/8aa9fe2d4303426c92f8fcfe9d36d542 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/8aa9fe2d4303426c92f8fcfe9d36d542 2024-11-21T13:24:21,770 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/da7154480a04485b990e8c00f5c7f25b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/da7154480a04485b990e8c00f5c7f25b 2024-11-21T13:24:21,771 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/b59e2652aac749ddbafbf3494c5bcbb1 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/b59e2652aac749ddbafbf3494c5bcbb1 2024-11-21T13:24:21,772 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/615001c099d940cb98bbfe9c534f4110 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/615001c099d940cb98bbfe9c534f4110 2024-11-21T13:24:21,773 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/678f34b5e444437388953730b8ef47d9 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/678f34b5e444437388953730b8ef47d9 2024-11-21T13:24:21,775 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/6eb563398bfa49ccad161e048014d103 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/6eb563398bfa49ccad161e048014d103 2024-11-21T13:24:21,776 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/79f78da77f034c5b9c68cbdf28e9a77c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/79f78da77f034c5b9c68cbdf28e9a77c 2024-11-21T13:24:21,777 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/5010acd118b345f5835738ade9a0ab14 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/5010acd118b345f5835738ade9a0ab14 2024-11-21T13:24:21,778 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/06b30ec2da774465b14be3930ae05ed3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/06b30ec2da774465b14be3930ae05ed3 2024-11-21T13:24:21,780 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/48cf8bc4d8074c49abc6b23a62adc739 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/48cf8bc4d8074c49abc6b23a62adc739 2024-11-21T13:24:21,781 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/744a7dccd7ba44a9aae646d6bcca9af7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/744a7dccd7ba44a9aae646d6bcca9af7 2024-11-21T13:24:21,783 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/1f4fa24a3c024ea88ec824c65e33ec4a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/1f4fa24a3c024ea88ec824c65e33ec4a 2024-11-21T13:24:21,784 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/28781f68c92f419ab0074195ed2658e2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/28781f68c92f419ab0074195ed2658e2 2024-11-21T13:24:21,786 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/f1b69f15e7c94c6eaed2acfb60ab8632 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/f1b69f15e7c94c6eaed2acfb60ab8632 2024-11-21T13:24:21,787 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ed0d0ceb639f4133b172f199fe6886bf to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ed0d0ceb639f4133b172f199fe6886bf 2024-11-21T13:24:21,788 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/2d6d98afaeb94fd881cd70afddf10d9c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/2d6d98afaeb94fd881cd70afddf10d9c 2024-11-21T13:24:21,789 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/38f94365e95a47a8ae1790eb7442fc30 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/38f94365e95a47a8ae1790eb7442fc30 2024-11-21T13:24:21,791 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/c6b7a13da5a04ae5a681cd8338fed9c8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/c6b7a13da5a04ae5a681cd8338fed9c8 2024-11-21T13:24:21,792 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/eefcd5af46c341a78684b6dc9e9284e1 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/eefcd5af46c341a78684b6dc9e9284e1 2024-11-21T13:24:21,793 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ac956bf3d278468daef3095b687c2afe to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ac956bf3d278468daef3095b687c2afe 2024-11-21T13:24:21,794 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/fe9968c9eb0c46ca86cdc46762de491d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/fe9968c9eb0c46ca86cdc46762de491d 2024-11-21T13:24:21,795 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/a2d4a7a1251f44049535e6f467189b3b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/a2d4a7a1251f44049535e6f467189b3b 2024-11-21T13:24:21,796 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/acf3fefc11c44367a169e4b8be4b685d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/acf3fefc11c44367a169e4b8be4b685d 2024-11-21T13:24:21,798 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/dee4f6a9954544778b2f93623119b5bb to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/dee4f6a9954544778b2f93623119b5bb 2024-11-21T13:24:21,799 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/3066bde85d5d4ee592ce6abafa6fe7e5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/3066bde85d5d4ee592ce6abafa6fe7e5 2024-11-21T13:24:21,810 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/b263dc6c504e4248a0db3001d8ee9e5e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bf37ed443e134eb4a44fda493468514c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/66099bbe30d0468da4df28f9dd8a9684, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/b1aa6a7e0c60480d9ac49a343dd01458, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a2cce96de17240bcb9cf9ea362095476, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/f48246852438427fb3a483c89237e725, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/d554dc7694264b149e236115326de517, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/7a9baffe5d7145f09da9e7a5ad54e447, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a119cd4822a84561940fbbbf66d6aee3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/c60015aee08749e299d85908239fb45a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/7ae9c9e8b8704a77aa4a55f150eeb145, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/62f2809bb366453d9f56765a765e752f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/1e56050c07e54a55adeafa1957a47254, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/00c71c8aa2984f8ea9e87a77d9e6a87a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/dba46caa194c4d8397d56eb7802974ed, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a9cbc2d29f1e41cd8800e38a1e78e644, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/879471d8b9c1497ab6b30ef0edd8c59f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/aa0f37a38c2a4cbda0d87eae3163d308, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0953fea064944a83b5682cdd5292acdd, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/e9c7b4a80ddf49358ae4f26f32751d01, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/cf088bad836a45b49dd96b92f169e1ee, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/1df39bebbf27407286aa6386e18ba945, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/393c24602f4b4b6e8bc6df2cfd00b35e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0638e4f4547848cbbcc1691b66400112, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/3783f1b3592c4d5ea696807297b7a7ac, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bfbc94a543394eb39d3abe3625e56ac8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/af83a242b4804ab298a73c73a659e83a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/cc1151fc93024f2fbc7aeab8a19565ec, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/02b5d8b44d7b446ba6b2b923635fa10f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a98615bc9af34aa9b1c83c4c8fc6b56f] to archive 2024-11-21T13:24:21,811 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:24:21,812 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/b263dc6c504e4248a0db3001d8ee9e5e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/b263dc6c504e4248a0db3001d8ee9e5e 2024-11-21T13:24:21,813 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bf37ed443e134eb4a44fda493468514c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bf37ed443e134eb4a44fda493468514c 2024-11-21T13:24:21,814 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/66099bbe30d0468da4df28f9dd8a9684 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/66099bbe30d0468da4df28f9dd8a9684 2024-11-21T13:24:21,815 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/b1aa6a7e0c60480d9ac49a343dd01458 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/b1aa6a7e0c60480d9ac49a343dd01458 2024-11-21T13:24:21,816 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a2cce96de17240bcb9cf9ea362095476 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a2cce96de17240bcb9cf9ea362095476 2024-11-21T13:24:21,817 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/f48246852438427fb3a483c89237e725 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/f48246852438427fb3a483c89237e725 2024-11-21T13:24:21,818 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/d554dc7694264b149e236115326de517 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/d554dc7694264b149e236115326de517 2024-11-21T13:24:21,819 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/7a9baffe5d7145f09da9e7a5ad54e447 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/7a9baffe5d7145f09da9e7a5ad54e447 2024-11-21T13:24:21,820 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a119cd4822a84561940fbbbf66d6aee3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a119cd4822a84561940fbbbf66d6aee3 2024-11-21T13:24:21,821 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/c60015aee08749e299d85908239fb45a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/c60015aee08749e299d85908239fb45a 2024-11-21T13:24:21,822 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/7ae9c9e8b8704a77aa4a55f150eeb145 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/7ae9c9e8b8704a77aa4a55f150eeb145 2024-11-21T13:24:21,823 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/62f2809bb366453d9f56765a765e752f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/62f2809bb366453d9f56765a765e752f 2024-11-21T13:24:21,824 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/1e56050c07e54a55adeafa1957a47254 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/1e56050c07e54a55adeafa1957a47254 2024-11-21T13:24:21,825 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/00c71c8aa2984f8ea9e87a77d9e6a87a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/00c71c8aa2984f8ea9e87a77d9e6a87a 2024-11-21T13:24:21,826 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/dba46caa194c4d8397d56eb7802974ed to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/dba46caa194c4d8397d56eb7802974ed 2024-11-21T13:24:21,827 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a9cbc2d29f1e41cd8800e38a1e78e644 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a9cbc2d29f1e41cd8800e38a1e78e644 2024-11-21T13:24:21,828 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/879471d8b9c1497ab6b30ef0edd8c59f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/879471d8b9c1497ab6b30ef0edd8c59f 2024-11-21T13:24:21,829 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/aa0f37a38c2a4cbda0d87eae3163d308 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/aa0f37a38c2a4cbda0d87eae3163d308 2024-11-21T13:24:21,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0953fea064944a83b5682cdd5292acdd to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0953fea064944a83b5682cdd5292acdd 2024-11-21T13:24:21,830 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/e9c7b4a80ddf49358ae4f26f32751d01 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/e9c7b4a80ddf49358ae4f26f32751d01 2024-11-21T13:24:21,831 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/cf088bad836a45b49dd96b92f169e1ee to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/cf088bad836a45b49dd96b92f169e1ee 2024-11-21T13:24:21,832 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/1df39bebbf27407286aa6386e18ba945 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/1df39bebbf27407286aa6386e18ba945 2024-11-21T13:24:21,833 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/393c24602f4b4b6e8bc6df2cfd00b35e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/393c24602f4b4b6e8bc6df2cfd00b35e 2024-11-21T13:24:21,834 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0638e4f4547848cbbcc1691b66400112 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0638e4f4547848cbbcc1691b66400112 2024-11-21T13:24:21,835 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/3783f1b3592c4d5ea696807297b7a7ac to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/3783f1b3592c4d5ea696807297b7a7ac 2024-11-21T13:24:21,836 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bfbc94a543394eb39d3abe3625e56ac8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bfbc94a543394eb39d3abe3625e56ac8 2024-11-21T13:24:21,837 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/af83a242b4804ab298a73c73a659e83a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/af83a242b4804ab298a73c73a659e83a 2024-11-21T13:24:21,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/cc1151fc93024f2fbc7aeab8a19565ec to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/cc1151fc93024f2fbc7aeab8a19565ec 2024-11-21T13:24:21,838 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/02b5d8b44d7b446ba6b2b923635fa10f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/02b5d8b44d7b446ba6b2b923635fa10f 2024-11-21T13:24:21,839 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a98615bc9af34aa9b1c83c4c8fc6b56f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/a98615bc9af34aa9b1c83c4c8fc6b56f 2024-11-21T13:24:21,841 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/c1ba87c9e4954cdb804ec2d34fb71310, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/4534e7cc58164897b98d56a4dab94737, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e6b894daaca4404486c60cc7929e68c5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b402ef22736549cf9e8e04ce513a1a29, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/2626b67585944674921aaf7e3c1c9648, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/917066414d6d4e21ae22f4df89809ab5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/1b4ee870741c433b808621995918aa89, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e30cbc7e37e7432b96e58eba821ee084, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/64148b4a052c4daa877ed9276f329451, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/dd0bc0358fdf491a8a5940b439356574, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/71ba5538ced74be6b64342202b3e00b0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/34aeef11d584439ab2458465869d9270, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5e2186c9ca0048b699ef02f87441d0d2, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/9c1c789b2caf458bb71c01b8e9fd19c2, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/07590f08a1254ee6b788d9d212dfe19d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5d9df3b9411e4809a45efa3c62cdf9ff, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/64451b08721d404f99a97852bb42fb59, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5283ae31a9664e95a8fbb00fa1dab45e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/ffe1bb9a9095444e8e7cd2a6cd6a1ae0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/3a53bd2b147d4998bd0ab8507766da6d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/791ac545bd804eaf9d5420653c6635c4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e9d7560e846a4cefa10fa4b7731304f6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e470acee75704e018fd17501f289a101, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/4b8ad7bb75d148c9a1dfca6ae5e23cfd, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5a63764cf56a434fa2af97f8c6d2efa0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b9c40090232e43688cced95d9beee657, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/bd262f7434db4176a9a44b165a9faa3d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b188b095a15a4b4bb4b9ee144a5cc699, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/962575d68d3e453c90e6d5bc3ca19ae7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e04c82babe8e4c3eb6b2abf20a44df8b] to archive 2024-11-21T13:24:21,842 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:24:21,843 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/c1ba87c9e4954cdb804ec2d34fb71310 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/c1ba87c9e4954cdb804ec2d34fb71310 2024-11-21T13:24:21,844 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/4534e7cc58164897b98d56a4dab94737 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/4534e7cc58164897b98d56a4dab94737 2024-11-21T13:24:21,845 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e6b894daaca4404486c60cc7929e68c5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e6b894daaca4404486c60cc7929e68c5 2024-11-21T13:24:21,846 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b402ef22736549cf9e8e04ce513a1a29 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b402ef22736549cf9e8e04ce513a1a29 2024-11-21T13:24:21,847 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/2626b67585944674921aaf7e3c1c9648 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/2626b67585944674921aaf7e3c1c9648 2024-11-21T13:24:21,848 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/917066414d6d4e21ae22f4df89809ab5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/917066414d6d4e21ae22f4df89809ab5 2024-11-21T13:24:21,849 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/1b4ee870741c433b808621995918aa89 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/1b4ee870741c433b808621995918aa89 2024-11-21T13:24:21,850 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e30cbc7e37e7432b96e58eba821ee084 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e30cbc7e37e7432b96e58eba821ee084 2024-11-21T13:24:21,851 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/64148b4a052c4daa877ed9276f329451 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/64148b4a052c4daa877ed9276f329451 2024-11-21T13:24:21,852 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/dd0bc0358fdf491a8a5940b439356574 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/dd0bc0358fdf491a8a5940b439356574 2024-11-21T13:24:21,853 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/71ba5538ced74be6b64342202b3e00b0 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/71ba5538ced74be6b64342202b3e00b0 2024-11-21T13:24:21,854 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/34aeef11d584439ab2458465869d9270 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/34aeef11d584439ab2458465869d9270 2024-11-21T13:24:21,855 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5e2186c9ca0048b699ef02f87441d0d2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5e2186c9ca0048b699ef02f87441d0d2 2024-11-21T13:24:21,856 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/9c1c789b2caf458bb71c01b8e9fd19c2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/9c1c789b2caf458bb71c01b8e9fd19c2 2024-11-21T13:24:21,857 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/07590f08a1254ee6b788d9d212dfe19d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/07590f08a1254ee6b788d9d212dfe19d 2024-11-21T13:24:21,858 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5d9df3b9411e4809a45efa3c62cdf9ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5d9df3b9411e4809a45efa3c62cdf9ff 2024-11-21T13:24:21,859 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/64451b08721d404f99a97852bb42fb59 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/64451b08721d404f99a97852bb42fb59 2024-11-21T13:24:21,860 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5283ae31a9664e95a8fbb00fa1dab45e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5283ae31a9664e95a8fbb00fa1dab45e 2024-11-21T13:24:21,861 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/ffe1bb9a9095444e8e7cd2a6cd6a1ae0 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/ffe1bb9a9095444e8e7cd2a6cd6a1ae0 2024-11-21T13:24:21,862 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/3a53bd2b147d4998bd0ab8507766da6d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/3a53bd2b147d4998bd0ab8507766da6d 2024-11-21T13:24:21,863 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/791ac545bd804eaf9d5420653c6635c4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/791ac545bd804eaf9d5420653c6635c4 2024-11-21T13:24:21,864 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e9d7560e846a4cefa10fa4b7731304f6 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e9d7560e846a4cefa10fa4b7731304f6 2024-11-21T13:24:21,865 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e470acee75704e018fd17501f289a101 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e470acee75704e018fd17501f289a101 2024-11-21T13:24:21,866 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/4b8ad7bb75d148c9a1dfca6ae5e23cfd to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/4b8ad7bb75d148c9a1dfca6ae5e23cfd 2024-11-21T13:24:21,867 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5a63764cf56a434fa2af97f8c6d2efa0 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5a63764cf56a434fa2af97f8c6d2efa0 2024-11-21T13:24:21,868 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b9c40090232e43688cced95d9beee657 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b9c40090232e43688cced95d9beee657 2024-11-21T13:24:21,869 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/bd262f7434db4176a9a44b165a9faa3d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/bd262f7434db4176a9a44b165a9faa3d 2024-11-21T13:24:21,870 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b188b095a15a4b4bb4b9ee144a5cc699 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b188b095a15a4b4bb4b9ee144a5cc699 2024-11-21T13:24:21,871 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/962575d68d3e453c90e6d5bc3ca19ae7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/962575d68d3e453c90e6d5bc3ca19ae7 2024-11-21T13:24:21,872 DEBUG [StoreCloser-TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e04c82babe8e4c3eb6b2abf20a44df8b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e04c82babe8e4c3eb6b2abf20a44df8b 2024-11-21T13:24:21,877 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/recovered.edits/468.seqid, newMaxSeqId=468, maxSeqId=1 2024-11-21T13:24:21,878 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. 2024-11-21T13:24:21,878 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] regionserver.HRegion(1676): Region close journal for 78998c23688e9623a40dfb78a0d6fff8: Waiting for close lock at 1732195460477Running coprocessor pre-close hooks at 1732195460477Disabling compacts and flushes for region at 1732195460477Disabling writes for close at 1732195460477Obtaining lock to block concurrent updates at 1732195460478 (+1 ms)Preparing flush snapshotting stores in 78998c23688e9623a40dfb78a0d6fff8 at 1732195460478Finished memstore snapshotting TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732195460478Flushing stores of TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8. at 1732195460479 (+1 ms)Flushing 78998c23688e9623a40dfb78a0d6fff8/A: creating writer at 1732195460479Flushing 78998c23688e9623a40dfb78a0d6fff8/A: appending metadata at 1732195460484 (+5 ms)Flushing 78998c23688e9623a40dfb78a0d6fff8/A: closing flushed file at 1732195460484Flushing 78998c23688e9623a40dfb78a0d6fff8/B: creating writer at 1732195460905 (+421 ms)Flushing 78998c23688e9623a40dfb78a0d6fff8/B: appending metadata at 1732195460908 (+3 ms)Flushing 78998c23688e9623a40dfb78a0d6fff8/B: closing flushed file at 1732195460908Flushing 78998c23688e9623a40dfb78a0d6fff8/C: creating writer at 1732195461322 (+414 ms)Flushing 78998c23688e9623a40dfb78a0d6fff8/C: appending metadata at 1732195461326 (+4 ms)Flushing 78998c23688e9623a40dfb78a0d6fff8/C: closing flushed file at 1732195461326Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68f8a3a: reopening flushed file at 1732195461736 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bcd95fa: reopening flushed file at 1732195461743 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74fdf0ba: reopening flushed file at 1732195461749 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 78998c23688e9623a40dfb78a0d6fff8 in 1277ms, sequenceid=465, compaction requested=true at 1732195461754 (+5 ms)Writing region close event to WAL at 1732195461874 (+120 ms)Running coprocessor post-close hooks at 1732195461878 (+4 ms)Closed at 1732195461878 2024-11-21T13:24:21,881 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=28}] handler.UnassignRegionHandler(157): Closed 78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:21,881 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=78998c23688e9623a40dfb78a0d6fff8, regionState=CLOSED 2024-11-21T13:24:21,883 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; CloseRegionProcedure 78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:24:21,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-21T13:24:21,886 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseRegionProcedure 78998c23688e9623a40dfb78a0d6fff8, server=4fe6efa1e611,33573,1732195430851 in 1.5680 sec 2024-11-21T13:24:21,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-21T13:24:21,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=78998c23688e9623a40dfb78a0d6fff8, UNASSIGN in 1.5750 sec 2024-11-21T13:24:21,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-21T13:24:21,890 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5810 sec 2024-11-21T13:24:21,891 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195461891"}]},"ts":"1732195461891"} 2024-11-21T13:24:21,893 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T13:24:21,893 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T13:24:21,895 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5970 sec 2024-11-21T13:24:22,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-21T13:24:22,461 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:22,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T13:24:22,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:22,474 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=29, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:22,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-21T13:24:22,475 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=29, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:22,479 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:22,483 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/recovered.edits] 2024-11-21T13:24:22,487 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/499e872aa5d841e38935448c05c740a8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/499e872aa5d841e38935448c05c740a8 2024-11-21T13:24:22,488 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/5c27e9ce3ffe4c93b741020829d819e2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/5c27e9ce3ffe4c93b741020829d819e2 2024-11-21T13:24:22,490 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/6369842f2cb94976b415b0bb4db7e25a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/6369842f2cb94976b415b0bb4db7e25a 2024-11-21T13:24:22,491 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ffe5e8080cb148f190652e064fc73343 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/A/ffe5e8080cb148f190652e064fc73343 2024-11-21T13:24:22,495 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0ba195a474b74907be911f52bf9386ba to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/0ba195a474b74907be911f52bf9386ba 2024-11-21T13:24:22,496 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/9433ff0dd4be4b08aa5af233fdc782a2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/9433ff0dd4be4b08aa5af233fdc782a2 2024-11-21T13:24:22,498 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bb3dd289daac443c9ef7b0ece62f1da3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/bb3dd289daac443c9ef7b0ece62f1da3 2024-11-21T13:24:22,500 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/f7dd767ae6a6469e9ba3834c098e0d73 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/B/f7dd767ae6a6469e9ba3834c098e0d73 2024-11-21T13:24:22,504 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/20b6fa8d4f174df09950cbba09f0929d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/20b6fa8d4f174df09950cbba09f0929d 2024-11-21T13:24:22,506 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5d3e8a58bf51466f97252a9ccedb8ce2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/5d3e8a58bf51466f97252a9ccedb8ce2 2024-11-21T13:24:22,508 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b51a7a7c44dc49a9b390fb5dfc2a252c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/b51a7a7c44dc49a9b390fb5dfc2a252c 2024-11-21T13:24:22,510 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e7922fbebc2241cb80faa98a38847373 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/C/e7922fbebc2241cb80faa98a38847373 2024-11-21T13:24:22,514 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/recovered.edits/468.seqid to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8/recovered.edits/468.seqid 2024-11-21T13:24:22,514 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/78998c23688e9623a40dfb78a0d6fff8 2024-11-21T13:24:22,514 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T13:24:22,519 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=29, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:22,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-21T13:24:22,527 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T13:24:22,555 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T13:24:22,557 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=29, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:22,557 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T13:24:22,558 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732195462557"}]},"ts":"9223372036854775807"} 2024-11-21T13:24:22,563 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T13:24:22,563 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 78998c23688e9623a40dfb78a0d6fff8, NAME => 'TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T13:24:22,564 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T13:24:22,564 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732195462564"}]},"ts":"9223372036854775807"} 2024-11-21T13:24:22,567 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T13:24:22,568 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=29, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:22,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-21T13:24:22,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 98 msec 2024-11-21T13:24:22,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-21T13:24:22,793 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T13:24:22,793 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:22,807 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=239 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_572311665_22 at /127.0.0.1:49182 [Waiting for operation #300] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1640571641_22 at /127.0.0.1:41840 [Waiting for operation #278] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;4fe6efa1e611:33573-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1640571641_22 at /127.0.0.1:53178 [Waiting for operation #52] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=607 (was 461) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2779 (was 3793) 2024-11-21T13:24:22,816 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=239, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=607, ProcessCount=11, AvailableMemoryMB=2778 2024-11-21T13:24:22,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T13:24:22,819 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T13:24:22,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:22,821 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T13:24:22,821 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:22,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 30 2024-11-21T13:24:22,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T13:24:22,823 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T13:24:22,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741939_1115 (size=959) 2024-11-21T13:24:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T13:24:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T13:24:23,236 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48 2024-11-21T13:24:23,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741940_1116 (size=53) 2024-11-21T13:24:23,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T13:24:23,646 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:24:23,646 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0eb6d933356cd6a9ba9fa27a04b9b8ff, disabling compactions & flushes 2024-11-21T13:24:23,646 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:23,646 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:23,646 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. after waiting 0 ms 2024-11-21T13:24:23,646 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:23,646 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:23,646 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: Waiting for close lock at 1732195463646Disabling compacts and flushes for region at 1732195463646Disabling writes for close at 1732195463646Writing region close event to WAL at 1732195463646Closed at 1732195463646 2024-11-21T13:24:23,648 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T13:24:23,649 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732195463649"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732195463649"}]},"ts":"1732195463649"} 2024-11-21T13:24:23,653 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T13:24:23,654 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T13:24:23,654 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195463654"}]},"ts":"1732195463654"} 2024-11-21T13:24:23,656 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T13:24:23,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, ASSIGN}] 2024-11-21T13:24:23,658 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, ASSIGN 2024-11-21T13:24:23,658 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, ASSIGN; state=OFFLINE, location=4fe6efa1e611,33573,1732195430851; forceNewPlan=false, retain=false 2024-11-21T13:24:23,809 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=0eb6d933356cd6a9ba9fa27a04b9b8ff, regionState=OPENING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:23,811 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, ASSIGN because future has completed 2024-11-21T13:24:23,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:24:23,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T13:24:23,969 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:23,969 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} 2024-11-21T13:24:23,969 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:23,970 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:24:23,970 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:23,970 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:23,972 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:23,976 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0eb6d933356cd6a9ba9fa27a04b9b8ff columnFamilyName A 2024-11-21T13:24:23,976 DEBUG [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:23,977 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] regionserver.HStore(327): Store=0eb6d933356cd6a9ba9fa27a04b9b8ff/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:24:23,977 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:23,979 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0eb6d933356cd6a9ba9fa27a04b9b8ff columnFamilyName B 2024-11-21T13:24:23,979 DEBUG [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:23,980 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] regionserver.HStore(327): Store=0eb6d933356cd6a9ba9fa27a04b9b8ff/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:24:23,980 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:23,981 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0eb6d933356cd6a9ba9fa27a04b9b8ff columnFamilyName C 2024-11-21T13:24:23,981 DEBUG [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:23,982 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] regionserver.HStore(327): Store=0eb6d933356cd6a9ba9fa27a04b9b8ff/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:24:23,982 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:23,983 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:23,983 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:23,984 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:23,984 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:23,985 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T13:24:23,986 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:23,993 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T13:24:23,994 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 0eb6d933356cd6a9ba9fa27a04b9b8ff; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71757580, jitterRate=0.0692712664604187}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T13:24:23,994 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:23,995 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: Running coprocessor pre-open hook at 1732195463970Writing region info on filesystem at 1732195463970Initializing all the Stores at 1732195463971 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195463971Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195463971Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195463971Cleaning up temporary data from old regions at 1732195463984 (+13 ms)Running coprocessor post-open hooks at 1732195463994 (+10 ms)Region opened successfully at 1732195463995 (+1 ms) 2024-11-21T13:24:23,996 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., pid=32, masterSystemTime=1732195463964 2024-11-21T13:24:23,998 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:23,998 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:23,999 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=0eb6d933356cd6a9ba9fa27a04b9b8ff, regionState=OPEN, openSeqNum=2, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:24,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:24:24,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-21T13:24:24,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 in 190 msec 2024-11-21T13:24:24,007 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-21T13:24:24,007 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, ASSIGN in 348 msec 2024-11-21T13:24:24,008 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T13:24:24,009 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195464008"}]},"ts":"1732195464008"} 2024-11-21T13:24:24,011 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T13:24:24,012 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=30, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T13:24:24,013 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1930 sec 2024-11-21T13:24:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=30 2024-11-21T13:24:24,970 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:24,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@642b414d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:24,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:24:24,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:24:24,975 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:24:24,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:24:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:24:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fcec947, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:24,976 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:24:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:24:24,977 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:24,979 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55136, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:24:24,980 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f289009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:24,981 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:24:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:24:24,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:24:24,983 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:24:24,984 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T13:24:24,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 4fe6efa1e611,46031,1732195430123 2024-11-21T13:24:24,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@281d24f1 2024-11-21T13:24:24,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T13:24:24,986 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T13:24:24,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T13:24:24,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T13:24:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:25,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741941_1117 (size=995) 2024-11-21T13:24:25,484 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T13:24:25,484 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T13:24:25,493 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T13:24:25,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, REOPEN/MOVE}] 2024-11-21T13:24:25,505 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, REOPEN/MOVE 2024-11-21T13:24:25,506 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=0eb6d933356cd6a9ba9fa27a04b9b8ff, regionState=CLOSING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:25,507 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, REOPEN/MOVE because future has completed 2024-11-21T13:24:25,508 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T13:24:25,508 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:24:25,662 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(122): Close 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:25,662 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T13:24:25,662 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1722): Closing 0eb6d933356cd6a9ba9fa27a04b9b8ff, disabling compactions & flushes 2024-11-21T13:24:25,662 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:25,662 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:25,663 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. after waiting 0 ms 2024-11-21T13:24:25,663 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:25,671 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T13:24:25,673 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:25,673 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegion(1676): Region close journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: Waiting for close lock at 1732195465662Running coprocessor pre-close hooks at 1732195465662Disabling compacts and flushes for region at 1732195465662Disabling writes for close at 1732195465663 (+1 ms)Writing region close event to WAL at 1732195465664 (+1 ms)Running coprocessor post-close hooks at 1732195465673 (+9 ms)Closed at 1732195465673 2024-11-21T13:24:25,673 WARN [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] regionserver.HRegionServer(3299): Not adding moved region record: 0eb6d933356cd6a9ba9fa27a04b9b8ff to self. 2024-11-21T13:24:25,676 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=36}] handler.UnassignRegionHandler(157): Closed 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:25,677 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=0eb6d933356cd6a9ba9fa27a04b9b8ff, regionState=CLOSED 2024-11-21T13:24:25,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:24:25,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-21T13:24:25,681 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; CloseRegionProcedure 0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 in 172 msec 2024-11-21T13:24:25,682 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, REOPEN/MOVE; state=CLOSED, location=4fe6efa1e611,33573,1732195430851; forceNewPlan=false, retain=true 2024-11-21T13:24:25,833 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=0eb6d933356cd6a9ba9fa27a04b9b8ff, regionState=OPENING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:25,838 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, REOPEN/MOVE because future has completed 2024-11-21T13:24:25,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:24:25,995 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:25,996 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} 2024-11-21T13:24:25,996 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:25,996 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:24:25,996 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:25,996 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,000 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,005 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0eb6d933356cd6a9ba9fa27a04b9b8ff columnFamilyName A 2024-11-21T13:24:26,007 DEBUG [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:26,007 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] regionserver.HStore(327): Store=0eb6d933356cd6a9ba9fa27a04b9b8ff/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:24:26,008 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,009 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0eb6d933356cd6a9ba9fa27a04b9b8ff columnFamilyName B 2024-11-21T13:24:26,009 DEBUG [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:26,009 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] regionserver.HStore(327): Store=0eb6d933356cd6a9ba9fa27a04b9b8ff/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:24:26,009 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,010 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0eb6d933356cd6a9ba9fa27a04b9b8ff columnFamilyName C 2024-11-21T13:24:26,010 DEBUG [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:26,010 INFO [StoreOpener-0eb6d933356cd6a9ba9fa27a04b9b8ff-1 {}] regionserver.HStore(327): Store=0eb6d933356cd6a9ba9fa27a04b9b8ff/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:24:26,011 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,011 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,012 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,013 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,013 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,014 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T13:24:26,015 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,016 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 0eb6d933356cd6a9ba9fa27a04b9b8ff; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60174203, jitterRate=-0.10333450138568878}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T13:24:26,016 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,017 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: Running coprocessor pre-open hook at 1732195465996Writing region info on filesystem at 1732195465996Initializing all the Stores at 1732195465997 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195465997Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195465997Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195465997Cleaning up temporary data from old regions at 1732195466013 (+16 ms)Running coprocessor post-open hooks at 1732195466016 (+3 ms)Region opened successfully at 1732195466017 (+1 ms) 2024-11-21T13:24:26,018 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., pid=37, masterSystemTime=1732195465992 2024-11-21T13:24:26,020 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:26,020 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:26,021 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=0eb6d933356cd6a9ba9fa27a04b9b8ff, regionState=OPEN, openSeqNum=5, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,022 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:24:26,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=35 2024-11-21T13:24:26,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 in 185 msec 2024-11-21T13:24:26,025 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-21T13:24:26,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, REOPEN/MOVE in 521 msec 2024-11-21T13:24:26,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-21T13:24:26,028 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 536 msec 2024-11-21T13:24:26,032 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 1.0360 sec 2024-11-21T13:24:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-21T13:24:26,034 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:26,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57e39aa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,034 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:24:26,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:24:26,035 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:24:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:24:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:24:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bc94d94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:24:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:24:26,036 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:26,037 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55156, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:24:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54337e63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,038 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3050598f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,038 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:24:26,038 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:24:26,039 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:24:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:24:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:24:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@478e196a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:24:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:24:26,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:26,041 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55172, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:24:26,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c0b21f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,042 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd99fb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,042 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:24:26,042 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:24:26,042 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:24:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:24:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:24:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ddadad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:24:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:24:26,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:26,044 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55186, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:24:26,044 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe1a835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,045 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b697fdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,045 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:24:26,045 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:24:26,046 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:24:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:24:26,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:24:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d21b58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:24:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:24:26,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:26,048 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:24:26,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65a747c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d441be3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,049 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:24:26,049 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:24:26,050 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:24:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:24:26,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:24:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1de9ba88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:24:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:24:26,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:26,052 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:24:26,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4150c50d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,053 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@799e2455, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,053 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:24:26,053 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:24:26,054 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:24:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:24:26,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:24:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4952c997, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:24:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:24:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:26,056 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55230, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:24:26,056 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@137d9b0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,057 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@115be4c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,057 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:24:26,057 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:24:26,058 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:24:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:24:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:24:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68926b50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,058 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:24:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:24:26,059 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:26,059 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55240, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:24:26,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a3eeec9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,061 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@621b6daf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,061 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:24:26,061 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:24:26,061 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:24:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:24:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:24:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b5b20b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:24:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:24:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:26,063 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55256, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:24:26,063 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e314a82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,064 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29007207, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,064 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:24:26,064 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:24:26,065 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:24:26,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:24:26,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:24:26,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5f8ded, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,065 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:24:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:24:26,066 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:26,066 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55280, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:24:26,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366baff1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:24:26,068 DEBUG [Thread-610 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:24:26,068 DEBUG [Thread-616 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:24:26,068 DEBUG [Thread-612 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:24:26,068 DEBUG [Thread-599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:24:26,068 DEBUG [Thread-601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:24:26,068 DEBUG [Thread-614 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:24:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:24:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:24:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:24:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:24:26,069 DEBUG [Thread-614 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:24:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:24:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:24:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:24:26,069 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:24:26,069 DEBUG [Thread-614 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:24:26,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:24:26,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:24:26,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:24:26,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:24:26,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55804, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:24:26,072 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:24:26,072 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:24:26,072 DEBUG [Thread-607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:24:26,072 DEBUG [Thread-603 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:24:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-21T13:24:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:24:26,074 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55814, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:24:26,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:24:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T13:24:26,074 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:26,074 DEBUG [Thread-607 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:24:26,075 DEBUG [Thread-607 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:24:26,075 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55828, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:24:26,075 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:26,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:24:26,076 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:24:26,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55838, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:24:26,078 DEBUG [Thread-605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:24:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:24:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:24:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:24:26,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:24:26,082 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:24:26,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:24:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:24:26,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:24:26,084 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:24:26,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:24:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,101 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:26,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:24:26,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c4f7c6bc23a7497aadd6fc3f076aa867_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195466085/Put/seqid=0 2024-11-21T13:24:26,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741942_1118 (size=17034) 2024-11-21T13:24:26,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195526159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195526163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195526163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,174 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195526171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195526165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,174 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,175 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,175 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T13:24:26,229 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T13:24:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:26,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:26,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195526279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195526280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195526280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195526280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195526280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,294 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T13:24:26,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:26,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:26,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:26,384 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T13:24:26,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195526500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195526500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195526500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,505 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195526500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195526501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T13:24:26,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:26,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:26,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:26,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,553 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:26,559 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c4f7c6bc23a7497aadd6fc3f076aa867_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c4f7c6bc23a7497aadd6fc3f076aa867_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,561 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/d8aa40d2379a4ae9873f5c34b8cb0be1, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:26,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/d8aa40d2379a4ae9873f5c34b8cb0be1 is 175, key is test_row_0/A:col10/1732195466085/Put/seqid=0 2024-11-21T13:24:26,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741943_1119 (size=48139) 2024-11-21T13:24:26,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/d8aa40d2379a4ae9873f5c34b8cb0be1 2024-11-21T13:24:26,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/c42da8269ef44cdfb9d61c62ef0f5add is 50, key is test_row_0/B:col10/1732195466085/Put/seqid=0 2024-11-21T13:24:26,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741944_1120 (size=12001) 2024-11-21T13:24:26,649 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/c42da8269ef44cdfb9d61c62ef0f5add 2024-11-21T13:24:26,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/12c4b107dbdc4ccebb44bf9133804bae is 50, key is test_row_0/C:col10/1732195466085/Put/seqid=0 2024-11-21T13:24:26,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741945_1121 (size=12001) 2024-11-21T13:24:26,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/12c4b107dbdc4ccebb44bf9133804bae 2024-11-21T13:24:26,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T13:24:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:26,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T13:24:26,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/d8aa40d2379a4ae9873f5c34b8cb0be1 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/d8aa40d2379a4ae9873f5c34b8cb0be1 2024-11-21T13:24:26,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/d8aa40d2379a4ae9873f5c34b8cb0be1, entries=250, sequenceid=17, filesize=47.0 K 2024-11-21T13:24:26,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/c42da8269ef44cdfb9d61c62ef0f5add as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c42da8269ef44cdfb9d61c62ef0f5add 2024-11-21T13:24:26,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c42da8269ef44cdfb9d61c62ef0f5add, entries=150, sequenceid=17, filesize=11.7 K 2024-11-21T13:24:26,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/12c4b107dbdc4ccebb44bf9133804bae as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/12c4b107dbdc4ccebb44bf9133804bae 2024-11-21T13:24:26,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/12c4b107dbdc4ccebb44bf9133804bae, entries=150, sequenceid=17, filesize=11.7 K 2024-11-21T13:24:26,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 654ms, sequenceid=17, compaction requested=false 2024-11-21T13:24:26,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:26,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,813 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:26,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T13:24:26,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112195184411d4d44b2782311a74d828a11d_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195466163/Put/seqid=0 2024-11-21T13:24:26,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195526830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195526831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195526833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,843 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195526833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195526835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,844 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741946_1122 (size=14594) 2024-11-21T13:24:26,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T13:24:26,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:26,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:26,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:26,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:26,853 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:26,862 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112195184411d4d44b2782311a74d828a11d_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112195184411d4d44b2782311a74d828a11d_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:26,863 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/a976bf05c8084b2aa2a387c53441126b, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:26,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/a976bf05c8084b2aa2a387c53441126b is 175, key is test_row_0/A:col10/1732195466163/Put/seqid=0 2024-11-21T13:24:26,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741947_1123 (size=39549) 2024-11-21T13:24:26,879 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/a976bf05c8084b2aa2a387c53441126b 2024-11-21T13:24:26,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/65a316cca9dc4e31b1d75a7a679e815b is 50, key is test_row_0/B:col10/1732195466163/Put/seqid=0 2024-11-21T13:24:26,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741948_1124 (size=12001) 2024-11-21T13:24:26,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/65a316cca9dc4e31b1d75a7a679e815b 2024-11-21T13:24:26,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/753a8eeb85ed4bd587a224198d015595 is 50, key is test_row_0/C:col10/1732195466163/Put/seqid=0 2024-11-21T13:24:26,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195526940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,954 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195526949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195526949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,955 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,955 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195526949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:26,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195526949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:26,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741949_1125 (size=12001) 2024-11-21T13:24:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:26,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:26,957 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/753a8eeb85ed4bd587a224198d015595 2024-11-21T13:24:26,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/a976bf05c8084b2aa2a387c53441126b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a976bf05c8084b2aa2a387c53441126b 2024-11-21T13:24:26,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a976bf05c8084b2aa2a387c53441126b, entries=200, sequenceid=41, filesize=38.6 K 2024-11-21T13:24:26,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/65a316cca9dc4e31b1d75a7a679e815b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/65a316cca9dc4e31b1d75a7a679e815b 2024-11-21T13:24:26,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/65a316cca9dc4e31b1d75a7a679e815b, entries=150, sequenceid=41, filesize=11.7 K 2024-11-21T13:24:26,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/753a8eeb85ed4bd587a224198d015595 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/753a8eeb85ed4bd587a224198d015595 2024-11-21T13:24:26,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/753a8eeb85ed4bd587a224198d015595, entries=150, sequenceid=41, filesize=11.7 K 2024-11-21T13:24:26,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 185ms, sequenceid=41, compaction requested=false 2024-11-21T13:24:26,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:27,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-21T13:24:27,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:27,003 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T13:24:27,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121de51a4df293a45d981f1d3c4f1fec7e5_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195466833/Put/seqid=0 2024-11-21T13:24:27,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741950_1126 (size=12154) 2024-11-21T13:24:27,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:27,088 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121de51a4df293a45d981f1d3c4f1fec7e5_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121de51a4df293a45d981f1d3c4f1fec7e5_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:27,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/1c4cd7ee778b43648cdc9973f0244e6d, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:27,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/1c4cd7ee778b43648cdc9973f0244e6d is 175, key is test_row_0/A:col10/1732195466833/Put/seqid=0 2024-11-21T13:24:27,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741951_1127 (size=30955) 2024-11-21T13:24:27,109 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/1c4cd7ee778b43648cdc9973f0244e6d 2024-11-21T13:24:27,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/114df3f6f44e4ea3811ae1f28a09b00a is 50, key is test_row_0/B:col10/1732195466833/Put/seqid=0 2024-11-21T13:24:27,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741952_1128 (size=12001) 2024-11-21T13:24:27,135 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/114df3f6f44e4ea3811ae1f28a09b00a 2024-11-21T13:24:27,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/39ce641f90774451849db558d3c6731e is 50, key is test_row_0/C:col10/1732195466833/Put/seqid=0 2024-11-21T13:24:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:27,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:27,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741953_1129 (size=12001) 2024-11-21T13:24:27,163 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/39ce641f90774451849db558d3c6731e 2024-11-21T13:24:27,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/1c4cd7ee778b43648cdc9973f0244e6d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1c4cd7ee778b43648cdc9973f0244e6d 2024-11-21T13:24:27,177 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1c4cd7ee778b43648cdc9973f0244e6d, entries=150, sequenceid=53, filesize=30.2 K 2024-11-21T13:24:27,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/114df3f6f44e4ea3811ae1f28a09b00a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/114df3f6f44e4ea3811ae1f28a09b00a 2024-11-21T13:24:27,185 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/114df3f6f44e4ea3811ae1f28a09b00a, entries=150, sequenceid=53, filesize=11.7 K 2024-11-21T13:24:27,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/39ce641f90774451849db558d3c6731e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/39ce641f90774451849db558d3c6731e 2024-11-21T13:24:27,202 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/39ce641f90774451849db558d3c6731e, entries=150, sequenceid=53, filesize=11.7 K 2024-11-21T13:24:27,203 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=87.22 KB/89310 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 200ms, sequenceid=53, compaction requested=true 2024-11-21T13:24:27,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:27,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:27,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-21T13:24:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-21T13:24:27,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-21T13:24:27,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1290 sec 2024-11-21T13:24:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T13:24:27,211 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:27,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T13:24:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:27,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.1380 sec 2024-11-21T13:24:27,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121abbb017edeeb4fe78695d4184bbd894c_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195467210/Put/seqid=0 2024-11-21T13:24:27,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741954_1130 (size=12154) 2024-11-21T13:24:27,246 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:27,251 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121abbb017edeeb4fe78695d4184bbd894c_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121abbb017edeeb4fe78695d4184bbd894c_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:27,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,253 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/6572e1482f58417fb7bbf620e8561099, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195527242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,254 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/6572e1482f58417fb7bbf620e8561099 is 175, key is test_row_0/A:col10/1732195467210/Put/seqid=0 2024-11-21T13:24:27,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195527243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195527245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195527250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195527253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,260 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741955_1131 (size=30955) 2024-11-21T13:24:27,275 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=71, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/6572e1482f58417fb7bbf620e8561099 2024-11-21T13:24:27,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b48a8f6dc1c04fb1b7a511ea3e5f4322 is 50, key is test_row_0/B:col10/1732195467210/Put/seqid=0 2024-11-21T13:24:27,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741956_1132 (size=12001) 2024-11-21T13:24:27,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b48a8f6dc1c04fb1b7a511ea3e5f4322 2024-11-21T13:24:27,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/c0533ac10f8d4193be616b38473999aa is 50, key is test_row_0/C:col10/1732195467210/Put/seqid=0 2024-11-21T13:24:27,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741957_1133 (size=12001) 2024-11-21T13:24:27,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/c0533ac10f8d4193be616b38473999aa 2024-11-21T13:24:27,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/6572e1482f58417fb7bbf620e8561099 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6572e1482f58417fb7bbf620e8561099 2024-11-21T13:24:27,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6572e1482f58417fb7bbf620e8561099, entries=150, sequenceid=71, filesize=30.2 K 2024-11-21T13:24:27,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b48a8f6dc1c04fb1b7a511ea3e5f4322 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b48a8f6dc1c04fb1b7a511ea3e5f4322 2024-11-21T13:24:27,349 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b48a8f6dc1c04fb1b7a511ea3e5f4322, entries=150, sequenceid=71, filesize=11.7 K 2024-11-21T13:24:27,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/c0533ac10f8d4193be616b38473999aa as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/c0533ac10f8d4193be616b38473999aa 2024-11-21T13:24:27,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195527359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195527359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195527359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/c0533ac10f8d4193be616b38473999aa, entries=150, sequenceid=71, filesize=11.7 K 2024-11-21T13:24:27,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 156ms, sequenceid=71, compaction requested=true 2024-11-21T13:24:27,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:27,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:27,367 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:27,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:27,368 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:27,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:27,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:27,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:27,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:27,371 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:27,371 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/A is initiating minor compaction (all files) 2024-11-21T13:24:27,371 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/A in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:27,371 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/d8aa40d2379a4ae9873f5c34b8cb0be1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a976bf05c8084b2aa2a387c53441126b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1c4cd7ee778b43648cdc9973f0244e6d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6572e1482f58417fb7bbf620e8561099] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=146.1 K 2024-11-21T13:24:27,371 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:27,372 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/d8aa40d2379a4ae9873f5c34b8cb0be1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a976bf05c8084b2aa2a387c53441126b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1c4cd7ee778b43648cdc9973f0244e6d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6572e1482f58417fb7bbf620e8561099] 2024-11-21T13:24:27,372 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8aa40d2379a4ae9873f5c34b8cb0be1, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732195466085 2024-11-21T13:24:27,373 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:27,373 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/B is initiating minor compaction (all files) 2024-11-21T13:24:27,373 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/B in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:27,373 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c42da8269ef44cdfb9d61c62ef0f5add, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/65a316cca9dc4e31b1d75a7a679e815b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/114df3f6f44e4ea3811ae1f28a09b00a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b48a8f6dc1c04fb1b7a511ea3e5f4322] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=46.9 K 2024-11-21T13:24:27,373 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting a976bf05c8084b2aa2a387c53441126b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732195466144 2024-11-21T13:24:27,374 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting c42da8269ef44cdfb9d61c62ef0f5add, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732195466085 2024-11-21T13:24:27,374 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c4cd7ee778b43648cdc9973f0244e6d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732195466830 2024-11-21T13:24:27,375 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 65a316cca9dc4e31b1d75a7a679e815b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732195466144 2024-11-21T13:24:27,375 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6572e1482f58417fb7bbf620e8561099, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732195467195 2024-11-21T13:24:27,376 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 114df3f6f44e4ea3811ae1f28a09b00a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732195466830 2024-11-21T13:24:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:27,376 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b48a8f6dc1c04fb1b7a511ea3e5f4322, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732195467195 2024-11-21T13:24:27,386 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:27,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T13:24:27,401 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#B#compaction#114 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:27,402 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/f567dd2203ad4149a666f3fd33bf1ddc is 50, key is test_row_0/B:col10/1732195467210/Put/seqid=0 2024-11-21T13:24:27,405 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:27,425 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411213d344a959bb049e38c35dc4c853791c6_0eb6d933356cd6a9ba9fa27a04b9b8ff store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:27,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195527426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,431 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195527427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b69469969e1142bc86f82d3aed89783f_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195467381/Put/seqid=0 2024-11-21T13:24:27,439 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411213d344a959bb049e38c35dc4c853791c6_0eb6d933356cd6a9ba9fa27a04b9b8ff, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:27,439 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213d344a959bb049e38c35dc4c853791c6_0eb6d933356cd6a9ba9fa27a04b9b8ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:27,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741958_1134 (size=12139) 2024-11-21T13:24:27,451 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/f567dd2203ad4149a666f3fd33bf1ddc as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/f567dd2203ad4149a666f3fd33bf1ddc 2024-11-21T13:24:27,456 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T13:24:27,458 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/B of 0eb6d933356cd6a9ba9fa27a04b9b8ff into f567dd2203ad4149a666f3fd33bf1ddc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:27,459 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:27,459 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/B, priority=12, startTime=1732195467367; duration=0sec 2024-11-21T13:24:27,459 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:27,459 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:B 2024-11-21T13:24:27,459 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:27,460 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:27,461 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/C is initiating minor compaction (all files) 2024-11-21T13:24:27,461 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/C in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:27,461 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/12c4b107dbdc4ccebb44bf9133804bae, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/753a8eeb85ed4bd587a224198d015595, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/39ce641f90774451849db558d3c6731e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/c0533ac10f8d4193be616b38473999aa] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=46.9 K 2024-11-21T13:24:27,461 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 12c4b107dbdc4ccebb44bf9133804bae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732195466085 2024-11-21T13:24:27,463 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 753a8eeb85ed4bd587a224198d015595, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732195466144 2024-11-21T13:24:27,463 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 39ce641f90774451849db558d3c6731e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1732195466830 2024-11-21T13:24:27,464 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting c0533ac10f8d4193be616b38473999aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732195467195 2024-11-21T13:24:27,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741959_1135 (size=14594) 2024-11-21T13:24:27,485 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:27,491 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b69469969e1142bc86f82d3aed89783f_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b69469969e1142bc86f82d3aed89783f_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:27,495 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/0cb76b156e2d43978a14bc24d99f216e, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:27,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/0cb76b156e2d43978a14bc24d99f216e is 175, key is test_row_0/A:col10/1732195467381/Put/seqid=0 2024-11-21T13:24:27,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741960_1136 (size=4469) 2024-11-21T13:24:27,508 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#A#compaction#115 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:27,508 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/7ed22395e0954790ba5c38b9cc359424 is 175, key is test_row_0/A:col10/1732195467210/Put/seqid=0 2024-11-21T13:24:27,513 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#C#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:27,513 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/051b1560c69c4a0e9ae94fca3ee208d8 is 50, key is test_row_0/C:col10/1732195467210/Put/seqid=0 2024-11-21T13:24:27,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195527540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,541 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,542 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195527540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,543 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741961_1137 (size=39549) 2024-11-21T13:24:27,545 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/0cb76b156e2d43978a14bc24d99f216e 2024-11-21T13:24:27,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741962_1138 (size=31093) 2024-11-21T13:24:27,556 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/7ed22395e0954790ba5c38b9cc359424 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/7ed22395e0954790ba5c38b9cc359424 2024-11-21T13:24:27,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741963_1139 (size=12139) 2024-11-21T13:24:27,564 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/A of 0eb6d933356cd6a9ba9fa27a04b9b8ff into 7ed22395e0954790ba5c38b9cc359424(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:27,564 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:27,565 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/A, priority=12, startTime=1732195467367; duration=0sec 2024-11-21T13:24:27,565 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:27,565 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:A 2024-11-21T13:24:27,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/ef2834da0d904edd90c4bd8a2ada69e5 is 50, key is test_row_0/B:col10/1732195467381/Put/seqid=0 2024-11-21T13:24:27,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195527570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195527570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195527570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,574 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,580 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/051b1560c69c4a0e9ae94fca3ee208d8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/051b1560c69c4a0e9ae94fca3ee208d8 2024-11-21T13:24:27,587 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/C of 0eb6d933356cd6a9ba9fa27a04b9b8ff into 051b1560c69c4a0e9ae94fca3ee208d8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:27,587 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:27,587 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/C, priority=12, startTime=1732195467368; duration=0sec 2024-11-21T13:24:27,587 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:27,587 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:C 2024-11-21T13:24:27,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741964_1140 (size=12001) 2024-11-21T13:24:27,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/ef2834da0d904edd90c4bd8a2ada69e5 2024-11-21T13:24:27,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/cf1403eecdb546269b9e7b108d1b4d95 is 50, key is test_row_0/C:col10/1732195467381/Put/seqid=0 2024-11-21T13:24:27,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741965_1141 (size=12001) 2024-11-21T13:24:27,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/cf1403eecdb546269b9e7b108d1b4d95 2024-11-21T13:24:27,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/0cb76b156e2d43978a14bc24d99f216e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/0cb76b156e2d43978a14bc24d99f216e 2024-11-21T13:24:27,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/0cb76b156e2d43978a14bc24d99f216e, entries=200, sequenceid=93, filesize=38.6 K 2024-11-21T13:24:27,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/ef2834da0d904edd90c4bd8a2ada69e5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/ef2834da0d904edd90c4bd8a2ada69e5 2024-11-21T13:24:27,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/ef2834da0d904edd90c4bd8a2ada69e5, entries=150, sequenceid=93, filesize=11.7 K 2024-11-21T13:24:27,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/cf1403eecdb546269b9e7b108d1b4d95 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/cf1403eecdb546269b9e7b108d1b4d95 2024-11-21T13:24:27,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/cf1403eecdb546269b9e7b108d1b4d95, entries=150, sequenceid=93, filesize=11.7 K 2024-11-21T13:24:27,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 265ms, sequenceid=93, compaction requested=false 2024-11-21T13:24:27,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:27,752 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:27,753 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T13:24:27,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112150840bd6d383489b8a7a98afcd70ac66_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195467423/Put/seqid=0 2024-11-21T13:24:27,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741966_1142 (size=14594) 2024-11-21T13:24:27,799 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:27,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195527793, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195527800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,803 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,809 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112150840bd6d383489b8a7a98afcd70ac66_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112150840bd6d383489b8a7a98afcd70ac66_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:27,810 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/e7233eef54474f05b4e4463be5407d77, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:27,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/e7233eef54474f05b4e4463be5407d77 is 175, key is test_row_0/A:col10/1732195467423/Put/seqid=0 2024-11-21T13:24:27,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741967_1143 (size=39549) 2024-11-21T13:24:27,825 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=111, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/e7233eef54474f05b4e4463be5407d77 2024-11-21T13:24:27,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/a3ada867e453428dbdc10c6123255025 is 50, key is test_row_0/B:col10/1732195467423/Put/seqid=0 2024-11-21T13:24:27,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741968_1144 (size=12001) 2024-11-21T13:24:27,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/a3ada867e453428dbdc10c6123255025 2024-11-21T13:24:27,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/3b50eb35490e42b4876ec4447e961a57 is 50, key is test_row_0/C:col10/1732195467423/Put/seqid=0 2024-11-21T13:24:27,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741969_1145 (size=12001) 2024-11-21T13:24:27,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=111 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/3b50eb35490e42b4876ec4447e961a57 2024-11-21T13:24:27,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195527880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195527880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195527880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/e7233eef54474f05b4e4463be5407d77 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e7233eef54474f05b4e4463be5407d77 2024-11-21T13:24:27,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e7233eef54474f05b4e4463be5407d77, entries=200, sequenceid=111, filesize=38.6 K 2024-11-21T13:24:27,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/a3ada867e453428dbdc10c6123255025 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/a3ada867e453428dbdc10c6123255025 2024-11-21T13:24:27,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/a3ada867e453428dbdc10c6123255025, entries=150, sequenceid=111, filesize=11.7 K 2024-11-21T13:24:27,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/3b50eb35490e42b4876ec4447e961a57 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/3b50eb35490e42b4876ec4447e961a57 2024-11-21T13:24:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195527909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:27,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195527909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:27,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:27,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/3b50eb35490e42b4876ec4447e961a57, entries=150, sequenceid=111, filesize=11.7 K 2024-11-21T13:24:27,926 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 174ms, sequenceid=111, compaction requested=true 2024-11-21T13:24:27,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:27,926 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:27,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:27,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:27,927 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:27,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:27,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:27,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:27,927 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:27,929 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:27,929 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/B is initiating minor compaction (all files) 2024-11-21T13:24:27,929 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/B in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:27,929 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/f567dd2203ad4149a666f3fd33bf1ddc, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/ef2834da0d904edd90c4bd8a2ada69e5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/a3ada867e453428dbdc10c6123255025] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=35.3 K 2024-11-21T13:24:27,930 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110191 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:27,930 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/A is initiating minor compaction (all files) 2024-11-21T13:24:27,930 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/A in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:27,930 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/7ed22395e0954790ba5c38b9cc359424, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/0cb76b156e2d43978a14bc24d99f216e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e7233eef54474f05b4e4463be5407d77] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=107.6 K 2024-11-21T13:24:27,930 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:27,930 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/7ed22395e0954790ba5c38b9cc359424, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/0cb76b156e2d43978a14bc24d99f216e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e7233eef54474f05b4e4463be5407d77] 2024-11-21T13:24:27,931 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f567dd2203ad4149a666f3fd33bf1ddc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732195467195 2024-11-21T13:24:27,931 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ed22395e0954790ba5c38b9cc359424, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732195467195 2024-11-21T13:24:27,931 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting ef2834da0d904edd90c4bd8a2ada69e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732195467374 2024-11-21T13:24:27,931 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cb76b156e2d43978a14bc24d99f216e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732195467249 2024-11-21T13:24:27,931 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting a3ada867e453428dbdc10c6123255025, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732195467423 2024-11-21T13:24:27,932 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7233eef54474f05b4e4463be5407d77, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732195467408 2024-11-21T13:24:27,944 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:27,944 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/145b3618371f4b63906e671fa3000329 is 50, key is test_row_0/B:col10/1732195467423/Put/seqid=0 2024-11-21T13:24:27,953 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:27,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741970_1146 (size=12241) 2024-11-21T13:24:27,972 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/145b3618371f4b63906e671fa3000329 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/145b3618371f4b63906e671fa3000329 2024-11-21T13:24:27,973 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121a08236dedfed4cb6bf16a11d254965b4_0eb6d933356cd6a9ba9fa27a04b9b8ff store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:27,976 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121a08236dedfed4cb6bf16a11d254965b4_0eb6d933356cd6a9ba9fa27a04b9b8ff, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:27,977 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a08236dedfed4cb6bf16a11d254965b4_0eb6d933356cd6a9ba9fa27a04b9b8ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:27,986 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/B of 0eb6d933356cd6a9ba9fa27a04b9b8ff into 145b3618371f4b63906e671fa3000329(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:27,987 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:27,987 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/B, priority=13, startTime=1732195467927; duration=0sec 2024-11-21T13:24:27,987 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:27,987 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:B 2024-11-21T13:24:27,987 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:27,994 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:27,994 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/C is initiating minor compaction (all files) 2024-11-21T13:24:27,994 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/C in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:27,994 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/051b1560c69c4a0e9ae94fca3ee208d8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/cf1403eecdb546269b9e7b108d1b4d95, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/3b50eb35490e42b4876ec4447e961a57] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=35.3 K 2024-11-21T13:24:27,994 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 051b1560c69c4a0e9ae94fca3ee208d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=71, earliestPutTs=1732195467195 2024-11-21T13:24:27,995 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting cf1403eecdb546269b9e7b108d1b4d95, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732195467374 2024-11-21T13:24:27,995 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b50eb35490e42b4876ec4447e961a57, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732195467423 2024-11-21T13:24:28,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741971_1147 (size=4469) 2024-11-21T13:24:28,010 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#A#compaction#124 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:28,011 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/9e303303884d48799b9760521641fe2f is 175, key is test_row_0/A:col10/1732195467423/Put/seqid=0 2024-11-21T13:24:28,015 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#C#compaction#125 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:28,015 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/500cc87e447c482783a73759a79deb3b is 50, key is test_row_0/C:col10/1732195467423/Put/seqid=0 2024-11-21T13:24:28,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741972_1148 (size=31195) 2024-11-21T13:24:28,036 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/9e303303884d48799b9760521641fe2f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/9e303303884d48799b9760521641fe2f 2024-11-21T13:24:28,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741973_1149 (size=12241) 2024-11-21T13:24:28,049 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/A of 0eb6d933356cd6a9ba9fa27a04b9b8ff into 9e303303884d48799b9760521641fe2f(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:28,050 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:28,050 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/A, priority=13, startTime=1732195467926; duration=0sec 2024-11-21T13:24:28,050 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:28,050 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:A 2024-11-21T13:24:28,052 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/500cc87e447c482783a73759a79deb3b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/500cc87e447c482783a73759a79deb3b 2024-11-21T13:24:28,060 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/C of 0eb6d933356cd6a9ba9fa27a04b9b8ff into 500cc87e447c482783a73759a79deb3b(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:28,061 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:28,061 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/C, priority=13, startTime=1732195467927; duration=0sec 2024-11-21T13:24:28,061 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:28,061 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:C 2024-11-21T13:24:28,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:28,123 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:28,123 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T13:24:28,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214f16c072576e43678238600945cc11f8_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195468121/Put/seqid=0 2024-11-21T13:24:28,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:28,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195528152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:28,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:28,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195528155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:28,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:28,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:28,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741974_1150 (size=17234) 2024-11-21T13:24:28,163 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:28,171 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214f16c072576e43678238600945cc11f8_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214f16c072576e43678238600945cc11f8_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:28,173 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/3664a84017734b2abe9527cd80dd56a6, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:28,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/3664a84017734b2abe9527cd80dd56a6 is 175, key is test_row_0/A:col10/1732195468121/Put/seqid=0 2024-11-21T13:24:28,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741975_1151 (size=48339) 2024-11-21T13:24:28,190 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/3664a84017734b2abe9527cd80dd56a6 2024-11-21T13:24:28,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/7318a7b7845443baabbe8b456146513c is 50, key is test_row_0/B:col10/1732195468121/Put/seqid=0 2024-11-21T13:24:28,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-21T13:24:28,220 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:28,220 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:28,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-21T13:24:28,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T13:24:28,223 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:28,224 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:28,224 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:28,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741976_1152 (size=12151) 2024-11-21T13:24:28,227 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/7318a7b7845443baabbe8b456146513c 2024-11-21T13:24:28,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/df5214e2de1e47989010ac9bf7b16404 is 50, key is test_row_0/C:col10/1732195468121/Put/seqid=0 2024-11-21T13:24:28,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741977_1153 (size=12151) 2024-11-21T13:24:28,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195528261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:28,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195528261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:28,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,263 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T13:24:28,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T13:24:28,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:28,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:28,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:28,376 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:28,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:28,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195528389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:28,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:28,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195528389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:28,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:28,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195528399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,402 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:28,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:28,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195528469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:28,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195528469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:28,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T13:24:28,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:28,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:28,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:28,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T13:24:28,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/df5214e2de1e47989010ac9bf7b16404 2024-11-21T13:24:28,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/3664a84017734b2abe9527cd80dd56a6 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/3664a84017734b2abe9527cd80dd56a6 2024-11-21T13:24:28,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/3664a84017734b2abe9527cd80dd56a6, entries=250, sequenceid=137, filesize=47.2 K 2024-11-21T13:24:28,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/7318a7b7845443baabbe8b456146513c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/7318a7b7845443baabbe8b456146513c 2024-11-21T13:24:28,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/7318a7b7845443baabbe8b456146513c, entries=150, sequenceid=137, filesize=11.9 K 2024-11-21T13:24:28,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/df5214e2de1e47989010ac9bf7b16404 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/df5214e2de1e47989010ac9bf7b16404 2024-11-21T13:24:28,679 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/df5214e2de1e47989010ac9bf7b16404, entries=150, sequenceid=137, filesize=11.9 K 2024-11-21T13:24:28,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 557ms, sequenceid=137, compaction requested=false 2024-11-21T13:24:28,680 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:28,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-21T13:24:28,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:28,683 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T13:24:28,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121858e4a2829634cbfa96a1353bfa1ccea_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195468153/Put/seqid=0 2024-11-21T13:24:28,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741978_1154 (size=12304) 2024-11-21T13:24:28,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:28,707 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121858e4a2829634cbfa96a1353bfa1ccea_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121858e4a2829634cbfa96a1353bfa1ccea_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:28,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/14df2e1660d943cfbe6c46dfeeaa5971, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:28,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/14df2e1660d943cfbe6c46dfeeaa5971 is 175, key is test_row_0/A:col10/1732195468153/Put/seqid=0 2024-11-21T13:24:28,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741979_1155 (size=31105) 2024-11-21T13:24:28,713 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=150, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/14df2e1660d943cfbe6c46dfeeaa5971 2024-11-21T13:24:28,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/9206e0af54604ebd89a504635dadf1ad is 50, key is test_row_0/B:col10/1732195468153/Put/seqid=0 2024-11-21T13:24:28,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741980_1156 (size=12151) 2024-11-21T13:24:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:28,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:28,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195528827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:28,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195528831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,835 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T13:24:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195528939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:28,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:28,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195528940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:28,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:29,132 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/9206e0af54604ebd89a504635dadf1ad 2024-11-21T13:24:29,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/068cad575c8b417ea21b5c083a9bcbee is 50, key is test_row_0/C:col10/1732195468153/Put/seqid=0 2024-11-21T13:24:29,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741981_1157 (size=12151) 2024-11-21T13:24:29,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:29,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195529149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:29,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195529149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:29,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T13:24:29,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:29,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195529399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:29,402 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 0eb6d933356cd6a9ba9fa27a04b9b8ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:29,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195529409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:29,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:29,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195529410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:29,410 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_1' in 0eb6d933356cd6a9ba9fa27a04b9b8ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:29,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,411 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 0eb6d933356cd6a9ba9fa27a04b9b8ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:29,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:29,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,411 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:29,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195529459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:29,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195529459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,462 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:29,545 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/068cad575c8b417ea21b5c083a9bcbee 2024-11-21T13:24:29,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/14df2e1660d943cfbe6c46dfeeaa5971 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/14df2e1660d943cfbe6c46dfeeaa5971 2024-11-21T13:24:29,554 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/14df2e1660d943cfbe6c46dfeeaa5971, entries=150, sequenceid=150, filesize=30.4 K 2024-11-21T13:24:29,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/9206e0af54604ebd89a504635dadf1ad as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/9206e0af54604ebd89a504635dadf1ad 2024-11-21T13:24:29,560 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/9206e0af54604ebd89a504635dadf1ad, entries=150, sequenceid=150, filesize=11.9 K 2024-11-21T13:24:29,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/068cad575c8b417ea21b5c083a9bcbee as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/068cad575c8b417ea21b5c083a9bcbee 2024-11-21T13:24:29,566 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/068cad575c8b417ea21b5c083a9bcbee, entries=150, sequenceid=150, filesize=11.9 K 2024-11-21T13:24:29,567 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 884ms, sequenceid=150, compaction requested=true 2024-11-21T13:24:29,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:29,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:29,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-21T13:24:29,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-21T13:24:29,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-21T13:24:29,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3450 sec 2024-11-21T13:24:29,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.3510 sec 2024-11-21T13:24:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:29,972 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:29,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T13:24:29,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218106792790434646a02673d9d0436531_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195468809/Put/seqid=0 2024-11-21T13:24:29,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:29,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195529988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:29,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195529989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:29,990 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:29,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741982_1158 (size=14794) 2024-11-21T13:24:29,998 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:30,006 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411218106792790434646a02673d9d0436531_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218106792790434646a02673d9d0436531_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:30,008 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/a027c13bbff9490c9d92f4a8b15a2aa8, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:30,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/a027c13bbff9490c9d92f4a8b15a2aa8 is 175, key is test_row_0/A:col10/1732195468809/Put/seqid=0 2024-11-21T13:24:30,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741983_1159 (size=39749) 2024-11-21T13:24:30,034 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=174, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/a027c13bbff9490c9d92f4a8b15a2aa8 2024-11-21T13:24:30,045 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/5a2d551e560f4991ba9b70f6021b2aea is 50, key is test_row_0/B:col10/1732195468809/Put/seqid=0 2024-11-21T13:24:30,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741984_1160 (size=12151) 2024-11-21T13:24:30,068 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/5a2d551e560f4991ba9b70f6021b2aea 2024-11-21T13:24:30,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/b476e9d113414282a721979259606f3e is 50, key is test_row_0/C:col10/1732195468809/Put/seqid=0 2024-11-21T13:24:30,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741985_1161 (size=12151) 2024-11-21T13:24:30,091 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=174 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/b476e9d113414282a721979259606f3e 2024-11-21T13:24:30,096 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/a027c13bbff9490c9d92f4a8b15a2aa8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a027c13bbff9490c9d92f4a8b15a2aa8 2024-11-21T13:24:30,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a027c13bbff9490c9d92f4a8b15a2aa8, entries=200, sequenceid=174, filesize=38.8 K 2024-11-21T13:24:30,103 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/5a2d551e560f4991ba9b70f6021b2aea as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/5a2d551e560f4991ba9b70f6021b2aea 2024-11-21T13:24:30,104 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:30,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195530100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:30,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:30,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:30,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195530100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:30,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,105 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:30,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/5a2d551e560f4991ba9b70f6021b2aea, entries=150, sequenceid=174, filesize=11.9 K 2024-11-21T13:24:30,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/b476e9d113414282a721979259606f3e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b476e9d113414282a721979259606f3e 2024-11-21T13:24:30,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b476e9d113414282a721979259606f3e, entries=150, sequenceid=174, filesize=11.9 K 2024-11-21T13:24:30,119 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 147ms, sequenceid=174, compaction requested=true 2024-11-21T13:24:30,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:30,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:30,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:30,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:30,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T13:24:30,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:30,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T13:24:30,120 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:30,120 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:30,122 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150388 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:30,122 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/A is initiating minor compaction (all files) 2024-11-21T13:24:30,122 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/A in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:30,122 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/9e303303884d48799b9760521641fe2f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/3664a84017734b2abe9527cd80dd56a6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/14df2e1660d943cfbe6c46dfeeaa5971, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a027c13bbff9490c9d92f4a8b15a2aa8] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=146.9 K 2024-11-21T13:24:30,122 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:30,122 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/9e303303884d48799b9760521641fe2f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/3664a84017734b2abe9527cd80dd56a6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/14df2e1660d943cfbe6c46dfeeaa5971, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a027c13bbff9490c9d92f4a8b15a2aa8] 2024-11-21T13:24:30,123 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:30,123 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/C is initiating minor compaction (all files) 2024-11-21T13:24:30,123 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/C in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:30,123 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/500cc87e447c482783a73759a79deb3b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/df5214e2de1e47989010ac9bf7b16404, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/068cad575c8b417ea21b5c083a9bcbee, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b476e9d113414282a721979259606f3e] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=47.6 K 2024-11-21T13:24:30,125 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 500cc87e447c482783a73759a79deb3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732195467423 2024-11-21T13:24:30,125 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e303303884d48799b9760521641fe2f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732195467423 2024-11-21T13:24:30,126 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting df5214e2de1e47989010ac9bf7b16404, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732195467791 2024-11-21T13:24:30,126 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 3664a84017734b2abe9527cd80dd56a6, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732195467776 2024-11-21T13:24:30,126 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 068cad575c8b417ea21b5c083a9bcbee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732195468146 2024-11-21T13:24:30,126 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 14df2e1660d943cfbe6c46dfeeaa5971, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732195468146 2024-11-21T13:24:30,126 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting b476e9d113414282a721979259606f3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732195468809 2024-11-21T13:24:30,126 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting a027c13bbff9490c9d92f4a8b15a2aa8, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732195468809 2024-11-21T13:24:30,138 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:30,144 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#C#compaction#136 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:30,144 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/145bfb3e4aee4a91839ceebe0e98c384 is 50, key is test_row_0/C:col10/1732195468809/Put/seqid=0 2024-11-21T13:24:30,149 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411217dc0101682814aa39c7edd7577ad9037_0eb6d933356cd6a9ba9fa27a04b9b8ff store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:30,151 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411217dc0101682814aa39c7edd7577ad9037_0eb6d933356cd6a9ba9fa27a04b9b8ff, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:30,152 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217dc0101682814aa39c7edd7577ad9037_0eb6d933356cd6a9ba9fa27a04b9b8ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:30,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741987_1163 (size=4469) 2024-11-21T13:24:30,175 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#A#compaction#135 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:30,176 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/fb178672fd544698b720197a2d9a1238 is 175, key is test_row_0/A:col10/1732195468809/Put/seqid=0 2024-11-21T13:24:30,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741986_1162 (size=12527) 2024-11-21T13:24:30,186 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/145bfb3e4aee4a91839ceebe0e98c384 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/145bfb3e4aee4a91839ceebe0e98c384 2024-11-21T13:24:30,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741988_1164 (size=31481) 2024-11-21T13:24:30,194 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/C of 0eb6d933356cd6a9ba9fa27a04b9b8ff into 145bfb3e4aee4a91839ceebe0e98c384(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:30,195 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:30,195 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/C, priority=12, startTime=1732195470120; duration=0sec 2024-11-21T13:24:30,195 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:30,195 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:C 2024-11-21T13:24:30,195 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:30,198 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:30,198 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/B is initiating minor compaction (all files) 2024-11-21T13:24:30,198 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/B in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:30,198 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/145b3618371f4b63906e671fa3000329, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/7318a7b7845443baabbe8b456146513c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/9206e0af54604ebd89a504635dadf1ad, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/5a2d551e560f4991ba9b70f6021b2aea] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=47.6 K 2024-11-21T13:24:30,199 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 145b3618371f4b63906e671fa3000329, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=111, earliestPutTs=1732195467423 2024-11-21T13:24:30,199 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7318a7b7845443baabbe8b456146513c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1732195467791 2024-11-21T13:24:30,200 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9206e0af54604ebd89a504635dadf1ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1732195468146 2024-11-21T13:24:30,200 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a2d551e560f4991ba9b70f6021b2aea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732195468809 2024-11-21T13:24:30,223 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#B#compaction#137 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:30,223 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/9d1240731985490faab39074f5fb54f3 is 50, key is test_row_0/B:col10/1732195468809/Put/seqid=0 2024-11-21T13:24:30,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741989_1165 (size=12527) 2024-11-21T13:24:30,254 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/9d1240731985490faab39074f5fb54f3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/9d1240731985490faab39074f5fb54f3 2024-11-21T13:24:30,265 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/B of 0eb6d933356cd6a9ba9fa27a04b9b8ff into 9d1240731985490faab39074f5fb54f3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:30,265 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:30,266 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/B, priority=12, startTime=1732195470120; duration=0sec 2024-11-21T13:24:30,266 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:30,266 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:B 2024-11-21T13:24:30,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:30,311 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:30,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T13:24:30,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121327b6f190f2e4f4aa67dc43e4ba156f7_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195469987/Put/seqid=0 2024-11-21T13:24:30,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741990_1166 (size=14794) 2024-11-21T13:24:30,329 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:30,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121327b6f190f2e4f4aa67dc43e4ba156f7_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121327b6f190f2e4f4aa67dc43e4ba156f7_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:30,341 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/4b2ae0e042004c7e9fdf3cf843babb6f, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:30,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/4b2ae0e042004c7e9fdf3cf843babb6f is 175, key is test_row_0/A:col10/1732195469987/Put/seqid=0 2024-11-21T13:24:30,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741991_1167 (size=39749) 2024-11-21T13:24:30,350 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/4b2ae0e042004c7e9fdf3cf843babb6f 2024-11-21T13:24:30,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195530362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-21T13:24:30,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:30,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195530367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:30,370 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:30,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/c01f76e27c7b4bd382e43f018d9b1d79 is 50, key is test_row_0/B:col10/1732195469987/Put/seqid=0 2024-11-21T13:24:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:30,371 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:30,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-21T13:24:30,374 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:30,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T13:24:30,375 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:30,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:30,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741992_1168 (size=12151) 2024-11-21T13:24:30,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/c01f76e27c7b4bd382e43f018d9b1d79 2024-11-21T13:24:30,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/026e72f93b6f42a99a3e0c3ed7600220 is 50, key is test_row_0/C:col10/1732195469987/Put/seqid=0 2024-11-21T13:24:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T13:24:30,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:30,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195530480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:30,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:30,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195530480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:30,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741993_1169 (size=12151) 2024-11-21T13:24:30,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T13:24:30,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:30,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:30,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:30,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:30,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:30,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:30,596 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/fb178672fd544698b720197a2d9a1238 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/fb178672fd544698b720197a2d9a1238 2024-11-21T13:24:30,606 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/A of 0eb6d933356cd6a9ba9fa27a04b9b8ff into fb178672fd544698b720197a2d9a1238(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:30,606 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:30,606 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/A, priority=12, startTime=1732195470120; duration=0sec 2024-11-21T13:24:30,607 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:30,607 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:A 2024-11-21T13:24:30,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T13:24:30,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:30,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:30,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:30,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:30,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T13:24:30,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195530689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195530689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:30,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:30,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:30,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T13:24:30,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:30,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:30,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:30,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:30,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:30,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:30,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/026e72f93b6f42a99a3e0c3ed7600220 2024-11-21T13:24:30,897 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/4b2ae0e042004c7e9fdf3cf843babb6f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/4b2ae0e042004c7e9fdf3cf843babb6f 2024-11-21T13:24:30,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/4b2ae0e042004c7e9fdf3cf843babb6f, entries=200, sequenceid=189, filesize=38.8 K 2024-11-21T13:24:30,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/c01f76e27c7b4bd382e43f018d9b1d79 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c01f76e27c7b4bd382e43f018d9b1d79 2024-11-21T13:24:30,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c01f76e27c7b4bd382e43f018d9b1d79, entries=150, sequenceid=189, filesize=11.9 K 2024-11-21T13:24:30,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/026e72f93b6f42a99a3e0c3ed7600220 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/026e72f93b6f42a99a3e0c3ed7600220 2024-11-21T13:24:30,932 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/026e72f93b6f42a99a3e0c3ed7600220, entries=150, sequenceid=189, filesize=11.9 K 2024-11-21T13:24:30,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 622ms, sequenceid=189, compaction requested=false 2024-11-21T13:24:30,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:30,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-21T13:24:30,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:30,992 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T13:24:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T13:24:31,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:31,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:31,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112108564d35e98541d1b2c83e2cff69a221_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195470364/Put/seqid=0 2024-11-21T13:24:31,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:31,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195531024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,029 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:31,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195531025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:31,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:31,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741994_1170 (size=12304) 2024-11-21T13:24:31,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:31,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195531139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:31,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:31,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195531139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,144 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:31,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195531350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:31,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:31,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195531350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:31,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,355 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:31,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:31,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195531419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:31,424 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 0eb6d933356cd6a9ba9fa27a04b9b8ff of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,425 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:31,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195531429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:31,432 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:31,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195531429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:31,432 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_1' in 0eb6d933356cd6a9ba9fa27a04b9b8ff of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,433 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:31,433 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 0eb6d933356cd6a9ba9fa27a04b9b8ff of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,434 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:31,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:31,453 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112108564d35e98541d1b2c83e2cff69a221_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112108564d35e98541d1b2c83e2cff69a221_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:31,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/c4a27d6eedb74976b588b45d5d6dc2ef, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:31,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/c4a27d6eedb74976b588b45d5d6dc2ef is 175, key is test_row_0/A:col10/1732195470364/Put/seqid=0 2024-11-21T13:24:31,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741995_1171 (size=31105) 2024-11-21T13:24:31,471 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/c4a27d6eedb74976b588b45d5d6dc2ef 2024-11-21T13:24:31,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T13:24:31,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/1224d006e8f14a1588b5a80d87aefcbc is 50, key is test_row_0/B:col10/1732195470364/Put/seqid=0 2024-11-21T13:24:31,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741996_1172 (size=12151) 2024-11-21T13:24:31,536 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/1224d006e8f14a1588b5a80d87aefcbc 2024-11-21T13:24:31,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/3b2e81ec6f8046969660a9950d7f69e4 is 50, key is test_row_0/C:col10/1732195470364/Put/seqid=0 2024-11-21T13:24:31,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741997_1173 (size=12151) 2024-11-21T13:24:31,594 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/3b2e81ec6f8046969660a9950d7f69e4 2024-11-21T13:24:31,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/c4a27d6eedb74976b588b45d5d6dc2ef as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c4a27d6eedb74976b588b45d5d6dc2ef 2024-11-21T13:24:31,615 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c4a27d6eedb74976b588b45d5d6dc2ef, entries=150, sequenceid=213, filesize=30.4 K 2024-11-21T13:24:31,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/1224d006e8f14a1588b5a80d87aefcbc as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/1224d006e8f14a1588b5a80d87aefcbc 2024-11-21T13:24:31,627 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/1224d006e8f14a1588b5a80d87aefcbc, entries=150, sequenceid=213, filesize=11.9 K 2024-11-21T13:24:31,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/3b2e81ec6f8046969660a9950d7f69e4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/3b2e81ec6f8046969660a9950d7f69e4 2024-11-21T13:24:31,635 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/3b2e81ec6f8046969660a9950d7f69e4, entries=150, sequenceid=213, filesize=11.9 K 2024-11-21T13:24:31,637 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 645ms, sequenceid=213, compaction requested=true 2024-11-21T13:24:31,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:31,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:31,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-21T13:24:31,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-21T13:24:31,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-21T13:24:31,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2640 sec 2024-11-21T13:24:31,644 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.2710 sec 2024-11-21T13:24:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:31,662 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:31,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T13:24:31,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c4f88c337db24f65a4aa4a23f99bf7cd_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195471660/Put/seqid=0 2024-11-21T13:24:31,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741998_1174 (size=14794) 2024-11-21T13:24:31,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:31,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:31,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195531716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:31,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195531717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:31,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:31,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195531829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:31,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195531830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:31,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:31,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:31,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:32,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:32,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195532039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:32,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:32,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:32,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:32,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:32,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195532039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:32,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:32,100 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:32,107 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c4f88c337db24f65a4aa4a23f99bf7cd_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c4f88c337db24f65a4aa4a23f99bf7cd_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:32,108 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/79f33788b9f54c49bcbc3d2688c3ca41, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:32,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/79f33788b9f54c49bcbc3d2688c3ca41 is 175, key is test_row_0/A:col10/1732195471660/Put/seqid=0 2024-11-21T13:24:32,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741999_1175 (size=39749) 2024-11-21T13:24:32,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:32,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195532360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:32,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195532360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:32,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:32,517 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/79f33788b9f54c49bcbc3d2688c3ca41 2024-11-21T13:24:32,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-21T13:24:32,519 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:32,519 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:32,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-21T13:24:32,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T13:24:32,521 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:32,522 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:32,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:32,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/1bcd4ca9880844269182e7095ee1a480 is 50, key is test_row_0/B:col10/1732195471660/Put/seqid=0 2024-11-21T13:24:32,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742000_1176 (size=12151) 2024-11-21T13:24:32,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/1bcd4ca9880844269182e7095ee1a480 2024-11-21T13:24:32,541 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/b97b376535d6445ca6280fdd9ba7af00 is 50, key is test_row_0/C:col10/1732195471660/Put/seqid=0 2024-11-21T13:24:32,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742001_1177 (size=12151) 2024-11-21T13:24:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T13:24:32,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T13:24:32,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:32,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:32,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:32,674 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:32,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:32,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T13:24:32,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:32,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:32,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:32,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:32,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:32,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T13:24:32,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:32,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195532869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:32,871 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:32,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:32,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195532869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:32,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:32,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:32,874 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:32,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/b97b376535d6445ca6280fdd9ba7af00 2024-11-21T13:24:32,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/79f33788b9f54c49bcbc3d2688c3ca41 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/79f33788b9f54c49bcbc3d2688c3ca41 2024-11-21T13:24:32,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/79f33788b9f54c49bcbc3d2688c3ca41, entries=200, sequenceid=227, filesize=38.8 K 2024-11-21T13:24:32,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/1bcd4ca9880844269182e7095ee1a480 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/1bcd4ca9880844269182e7095ee1a480 2024-11-21T13:24:32,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/1bcd4ca9880844269182e7095ee1a480, entries=150, sequenceid=227, filesize=11.9 K 2024-11-21T13:24:32,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/b97b376535d6445ca6280fdd9ba7af00 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b97b376535d6445ca6280fdd9ba7af00 2024-11-21T13:24:32,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b97b376535d6445ca6280fdd9ba7af00, entries=150, sequenceid=227, filesize=11.9 K 2024-11-21T13:24:32,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 1306ms, sequenceid=227, compaction requested=true 2024-11-21T13:24:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:32,969 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:32,969 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:32,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:32,971 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:32,971 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/B is initiating minor compaction (all files) 2024-11-21T13:24:32,971 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/B in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:32,971 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/9d1240731985490faab39074f5fb54f3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c01f76e27c7b4bd382e43f018d9b1d79, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/1224d006e8f14a1588b5a80d87aefcbc, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/1bcd4ca9880844269182e7095ee1a480] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=47.8 K 2024-11-21T13:24:32,971 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142084 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:32,971 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/A is initiating minor compaction (all files) 2024-11-21T13:24:32,971 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/A in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:32,971 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/fb178672fd544698b720197a2d9a1238, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/4b2ae0e042004c7e9fdf3cf843babb6f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c4a27d6eedb74976b588b45d5d6dc2ef, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/79f33788b9f54c49bcbc3d2688c3ca41] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=138.8 K 2024-11-21T13:24:32,971 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d1240731985490faab39074f5fb54f3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732195468809 2024-11-21T13:24:32,971 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:32,971 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/fb178672fd544698b720197a2d9a1238, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/4b2ae0e042004c7e9fdf3cf843babb6f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c4a27d6eedb74976b588b45d5d6dc2ef, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/79f33788b9f54c49bcbc3d2688c3ca41] 2024-11-21T13:24:32,972 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting c01f76e27c7b4bd382e43f018d9b1d79, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732195469983 2024-11-21T13:24:32,972 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb178672fd544698b720197a2d9a1238, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732195468809 2024-11-21T13:24:32,972 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 1224d006e8f14a1588b5a80d87aefcbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732195470346 2024-11-21T13:24:32,973 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bcd4ca9880844269182e7095ee1a480, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732195471021 2024-11-21T13:24:32,973 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b2ae0e042004c7e9fdf3cf843babb6f, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732195469978 2024-11-21T13:24:32,973 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4a27d6eedb74976b588b45d5d6dc2ef, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732195470346 2024-11-21T13:24:32,973 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79f33788b9f54c49bcbc3d2688c3ca41, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732195471019 2024-11-21T13:24:32,980 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:32,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-21T13:24:32,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:32,981 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T13:24:32,982 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#B#compaction#147 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:32,982 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/c24d845309d14746b282e650b7cdec8c is 50, key is test_row_0/B:col10/1732195471660/Put/seqid=0 2024-11-21T13:24:32,985 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121bf0a0e7327c2455f80765d319893c1bb_0eb6d933356cd6a9ba9fa27a04b9b8ff store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:32,988 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121bf0a0e7327c2455f80765d319893c1bb_0eb6d933356cd6a9ba9fa27a04b9b8ff, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:32,990 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121bf0a0e7327c2455f80765d319893c1bb_0eb6d933356cd6a9ba9fa27a04b9b8ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:32,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742002_1178 (size=12663) 2024-11-21T13:24:32,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121794c0a7f06b947838a7fafb3846a43fd_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195471709/Put/seqid=0 2024-11-21T13:24:33,001 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/c24d845309d14746b282e650b7cdec8c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c24d845309d14746b282e650b7cdec8c 2024-11-21T13:24:33,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742003_1179 (size=4469) 2024-11-21T13:24:33,013 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#A#compaction#148 average throughput is 0.74 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:33,013 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/e2ed7d73167244bda0b3941006555a77 is 175, key is test_row_0/A:col10/1732195471660/Put/seqid=0 2024-11-21T13:24:33,015 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/B of 0eb6d933356cd6a9ba9fa27a04b9b8ff into c24d845309d14746b282e650b7cdec8c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:33,015 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:33,015 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/B, priority=12, startTime=1732195472969; duration=0sec 2024-11-21T13:24:33,015 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:33,015 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:B 2024-11-21T13:24:33,015 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:33,017 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:33,017 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/C is initiating minor compaction (all files) 2024-11-21T13:24:33,017 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/C in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:33,017 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/145bfb3e4aee4a91839ceebe0e98c384, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/026e72f93b6f42a99a3e0c3ed7600220, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/3b2e81ec6f8046969660a9950d7f69e4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b97b376535d6445ca6280fdd9ba7af00] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=47.8 K 2024-11-21T13:24:33,018 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 145bfb3e4aee4a91839ceebe0e98c384, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=174, earliestPutTs=1732195468809 2024-11-21T13:24:33,021 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 026e72f93b6f42a99a3e0c3ed7600220, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1732195469983 2024-11-21T13:24:33,023 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b2e81ec6f8046969660a9950d7f69e4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732195470346 2024-11-21T13:24:33,024 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b97b376535d6445ca6280fdd9ba7af00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732195471021 2024-11-21T13:24:33,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742004_1180 (size=12304) 2024-11-21T13:24:33,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742005_1181 (size=31617) 2024-11-21T13:24:33,033 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#C#compaction#150 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:33,034 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/f260ca17a652430cbf03937a2f5111b1 is 50, key is test_row_0/C:col10/1732195471660/Put/seqid=0 2024-11-21T13:24:33,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742006_1182 (size=12663) 2024-11-21T13:24:33,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T13:24:33,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:33,432 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121794c0a7f06b947838a7fafb3846a43fd_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121794c0a7f06b947838a7fafb3846a43fd_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:33,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/580c6d53262c4b70ae69d080d90ab0ed, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:33,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/580c6d53262c4b70ae69d080d90ab0ed is 175, key is test_row_0/A:col10/1732195471709/Put/seqid=0 2024-11-21T13:24:33,442 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/e2ed7d73167244bda0b3941006555a77 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e2ed7d73167244bda0b3941006555a77 2024-11-21T13:24:33,449 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/f260ca17a652430cbf03937a2f5111b1 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/f260ca17a652430cbf03937a2f5111b1 2024-11-21T13:24:33,454 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/A of 0eb6d933356cd6a9ba9fa27a04b9b8ff into e2ed7d73167244bda0b3941006555a77(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:33,454 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:33,454 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/A, priority=12, startTime=1732195472969; duration=0sec 2024-11-21T13:24:33,454 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:33,454 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:A 2024-11-21T13:24:33,456 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/C of 0eb6d933356cd6a9ba9fa27a04b9b8ff into f260ca17a652430cbf03937a2f5111b1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:33,456 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:33,456 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/C, priority=12, startTime=1732195472969; duration=0sec 2024-11-21T13:24:33,456 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:33,457 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:C 2024-11-21T13:24:33,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742007_1183 (size=31105) 2024-11-21T13:24:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T13:24:33,863 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/580c6d53262c4b70ae69d080d90ab0ed 2024-11-21T13:24:33,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/598edc29499e40278afa9ee6c788a239 is 50, key is test_row_0/B:col10/1732195471709/Put/seqid=0 2024-11-21T13:24:33,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742008_1184 (size=12151) 2024-11-21T13:24:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:33,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:33,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195533906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:33,907 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:33,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195533906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:33,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:34,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:34,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195534009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:34,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:34,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195534019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:34,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:34,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195534219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:34,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:34,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195534230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,232 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:34,278 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/598edc29499e40278afa9ee6c788a239 2024-11-21T13:24:34,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/4de0be26117342c0a861f86ac9fc94a8 is 50, key is test_row_0/C:col10/1732195471709/Put/seqid=0 2024-11-21T13:24:34,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742009_1185 (size=12151) 2024-11-21T13:24:34,292 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/4de0be26117342c0a861f86ac9fc94a8 2024-11-21T13:24:34,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/580c6d53262c4b70ae69d080d90ab0ed as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/580c6d53262c4b70ae69d080d90ab0ed 2024-11-21T13:24:34,305 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/580c6d53262c4b70ae69d080d90ab0ed, entries=150, sequenceid=249, filesize=30.4 K 2024-11-21T13:24:34,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/598edc29499e40278afa9ee6c788a239 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/598edc29499e40278afa9ee6c788a239 2024-11-21T13:24:34,311 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/598edc29499e40278afa9ee6c788a239, entries=150, sequenceid=249, filesize=11.9 K 2024-11-21T13:24:34,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/4de0be26117342c0a861f86ac9fc94a8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4de0be26117342c0a861f86ac9fc94a8 2024-11-21T13:24:34,318 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4de0be26117342c0a861f86ac9fc94a8, entries=150, sequenceid=249, filesize=11.9 K 2024-11-21T13:24:34,321 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 1339ms, sequenceid=249, compaction requested=false 2024-11-21T13:24:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:34,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-21T13:24:34,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-21T13:24:34,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-21T13:24:34,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8010 sec 2024-11-21T13:24:34,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.8070 sec 2024-11-21T13:24:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:34,531 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:34,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T13:24:34,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112165aac69c4aa74e1791da87fcc21a1939_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195473904/Put/seqid=0 2024-11-21T13:24:34,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742010_1186 (size=14994) 2024-11-21T13:24:34,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:34,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195534568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:34,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:34,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195534569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,574 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:34,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-21T13:24:34,679 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:34,680 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-21T13:24:34,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T13:24:34,682 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:34,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195534679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,682 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,682 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:34,682 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:34,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195534679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:34,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:34,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:34,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T13:24:34,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T13:24:34,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:34,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:34,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:34,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:34,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:34,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:34,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195534889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:34,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:34,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195534889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:34,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:34,911 INFO [master/4fe6efa1e611:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-21T13:24:34,911 INFO [master/4fe6efa1e611:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-21T13:24:34,943 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:34,948 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112165aac69c4aa74e1791da87fcc21a1939_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112165aac69c4aa74e1791da87fcc21a1939_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:34,949 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/c077e4e7732e447c973f6bfd11ecfc46, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:34,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/c077e4e7732e447c973f6bfd11ecfc46 is 175, key is test_row_0/A:col10/1732195473904/Put/seqid=0 2024-11-21T13:24:34,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742011_1187 (size=39949) 2024-11-21T13:24:34,987 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=267, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/c077e4e7732e447c973f6bfd11ecfc46 2024-11-21T13:24:34,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T13:24:34,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:34,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:34,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:34,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:34,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:34,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T13:24:35,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/4dcf0d062c6148db9a226c12e43a17ff is 50, key is test_row_0/B:col10/1732195473904/Put/seqid=0 2024-11-21T13:24:35,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742012_1188 (size=12301) 2024-11-21T13:24:35,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T13:24:35,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:35,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:35,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:35,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195535236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:35,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:35,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195535236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:35,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:35,237 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:35,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T13:24:35,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:35,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:35,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:35,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T13:24:35,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:35,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195535429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:35,437 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 0eb6d933356cd6a9ba9fa27a04b9b8ff of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:35,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T13:24:35,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:35,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:35,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:35,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195535459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:35,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195535459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:35,467 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_1' in 0eb6d933356cd6a9ba9fa27a04b9b8ff of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:35,468 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 0eb6d933356cd6a9ba9fa27a04b9b8ff of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:35,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/4dcf0d062c6148db9a226c12e43a17ff 2024-11-21T13:24:35,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/de1d11d939a24cd88a81d9c849135af9 is 50, key is test_row_0/C:col10/1732195473904/Put/seqid=0 2024-11-21T13:24:35,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742013_1189 (size=12301) 2024-11-21T13:24:35,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T13:24:35,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:35,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:35,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:35,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:35,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195535749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:35,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:35,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195535750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:35,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,753 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:35,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:35,753 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:35,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T13:24:35,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:35,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:35,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:35,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T13:24:35,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T13:24:35,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:35,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:35,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:35,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:35,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:36,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/de1d11d939a24cd88a81d9c849135af9 2024-11-21T13:24:36,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/c077e4e7732e447c973f6bfd11ecfc46 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c077e4e7732e447c973f6bfd11ecfc46 2024-11-21T13:24:36,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c077e4e7732e447c973f6bfd11ecfc46, entries=200, sequenceid=267, filesize=39.0 K 2024-11-21T13:24:36,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/4dcf0d062c6148db9a226c12e43a17ff as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/4dcf0d062c6148db9a226c12e43a17ff 2024-11-21T13:24:36,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/4dcf0d062c6148db9a226c12e43a17ff, entries=150, sequenceid=267, filesize=12.0 K 2024-11-21T13:24:36,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/de1d11d939a24cd88a81d9c849135af9 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/de1d11d939a24cd88a81d9c849135af9 2024-11-21T13:24:36,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/de1d11d939a24cd88a81d9c849135af9, entries=150, sequenceid=267, filesize=12.0 K 2024-11-21T13:24:36,057 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 1526ms, sequenceid=267, compaction requested=true 2024-11-21T13:24:36,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:36,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:36,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:36,058 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:36,058 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:36,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:36,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:36,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:36,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:36,063 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:36,063 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:36,063 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/B is initiating minor compaction (all files) 2024-11-21T13:24:36,063 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/A is initiating minor compaction (all files) 2024-11-21T13:24:36,063 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/B in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:36,063 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/A in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:36,063 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e2ed7d73167244bda0b3941006555a77, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/580c6d53262c4b70ae69d080d90ab0ed, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c077e4e7732e447c973f6bfd11ecfc46] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=100.3 K 2024-11-21T13:24:36,063 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c24d845309d14746b282e650b7cdec8c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/598edc29499e40278afa9ee6c788a239, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/4dcf0d062c6148db9a226c12e43a17ff] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=36.2 K 2024-11-21T13:24:36,063 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:36,063 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e2ed7d73167244bda0b3941006555a77, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/580c6d53262c4b70ae69d080d90ab0ed, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c077e4e7732e447c973f6bfd11ecfc46] 2024-11-21T13:24:36,063 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting c24d845309d14746b282e650b7cdec8c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732195471021 2024-11-21T13:24:36,064 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2ed7d73167244bda0b3941006555a77, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732195471021 2024-11-21T13:24:36,064 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 598edc29499e40278afa9ee6c788a239, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732195471707 2024-11-21T13:24:36,064 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 580c6d53262c4b70ae69d080d90ab0ed, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732195471707 2024-11-21T13:24:36,065 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dcf0d062c6148db9a226c12e43a17ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732195473904 2024-11-21T13:24:36,072 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting c077e4e7732e447c973f6bfd11ecfc46, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732195473900 2024-11-21T13:24:36,087 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:36,099 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#B#compaction#157 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:36,100 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b2fa84847ec746798dc1c3dfc1ce7a0b is 50, key is test_row_0/B:col10/1732195473904/Put/seqid=0 2024-11-21T13:24:36,101 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121112443735f6d464594d658d48458dc0a_0eb6d933356cd6a9ba9fa27a04b9b8ff store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:36,103 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121112443735f6d464594d658d48458dc0a_0eb6d933356cd6a9ba9fa27a04b9b8ff, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:36,103 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121112443735f6d464594d658d48458dc0a_0eb6d933356cd6a9ba9fa27a04b9b8ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:36,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-21T13:24:36,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:36,116 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T13:24:36,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742014_1190 (size=12915) 2024-11-21T13:24:36,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742015_1191 (size=4469) 2024-11-21T13:24:36,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216a3cd89c633742f88ecc1e867921785c_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195474567/Put/seqid=0 2024-11-21T13:24:36,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742016_1192 (size=12454) 2024-11-21T13:24:36,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,161 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216a3cd89c633742f88ecc1e867921785c_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216a3cd89c633742f88ecc1e867921785c_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:36,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/68b28ce3e63a40849e42abef0e592a5e, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:36,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/68b28ce3e63a40849e42abef0e592a5e is 175, key is test_row_0/A:col10/1732195474567/Put/seqid=0 2024-11-21T13:24:36,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742017_1193 (size=31255) 2024-11-21T13:24:36,173 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/68b28ce3e63a40849e42abef0e592a5e 2024-11-21T13:24:36,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/f4f1a25c37f843899ed0d585a0d4a62c is 50, key is test_row_0/B:col10/1732195474567/Put/seqid=0 2024-11-21T13:24:36,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742018_1194 (size=12301) 2024-11-21T13:24:36,192 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/f4f1a25c37f843899ed0d585a0d4a62c 2024-11-21T13:24:36,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/38005dc928624fffa63f77d67c2cd28f is 50, key is test_row_0/C:col10/1732195474567/Put/seqid=0 2024-11-21T13:24:36,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742019_1195 (size=12301) 2024-11-21T13:24:36,530 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#A#compaction#156 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:36,531 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/c680e970960e451d8d51751482c37ca3 is 175, key is test_row_0/A:col10/1732195473904/Put/seqid=0 2024-11-21T13:24:36,531 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b2fa84847ec746798dc1c3dfc1ce7a0b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b2fa84847ec746798dc1c3dfc1ce7a0b 2024-11-21T13:24:36,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742020_1196 (size=31869) 2024-11-21T13:24:36,538 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/B of 0eb6d933356cd6a9ba9fa27a04b9b8ff into b2fa84847ec746798dc1c3dfc1ce7a0b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:36,538 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:36,538 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/B, priority=13, startTime=1732195476058; duration=0sec 2024-11-21T13:24:36,538 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:36,538 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:B 2024-11-21T13:24:36,538 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:36,539 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:36,539 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/C is initiating minor compaction (all files) 2024-11-21T13:24:36,539 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/C in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:36,539 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/f260ca17a652430cbf03937a2f5111b1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4de0be26117342c0a861f86ac9fc94a8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/de1d11d939a24cd88a81d9c849135af9] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=36.2 K 2024-11-21T13:24:36,540 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f260ca17a652430cbf03937a2f5111b1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1732195471021 2024-11-21T13:24:36,540 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 4de0be26117342c0a861f86ac9fc94a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732195471707 2024-11-21T13:24:36,540 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/c680e970960e451d8d51751482c37ca3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c680e970960e451d8d51751482c37ca3 2024-11-21T13:24:36,540 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting de1d11d939a24cd88a81d9c849135af9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732195473904 2024-11-21T13:24:36,548 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/A of 0eb6d933356cd6a9ba9fa27a04b9b8ff into c680e970960e451d8d51751482c37ca3(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:36,548 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:36,548 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/A, priority=13, startTime=1732195476057; duration=0sec 2024-11-21T13:24:36,548 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:36,548 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:A 2024-11-21T13:24:36,549 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#C#compaction#161 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:36,550 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/e49f352c6e624248bb9f826ffbf12df4 is 50, key is test_row_0/C:col10/1732195473904/Put/seqid=0 2024-11-21T13:24:36,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742021_1197 (size=12915) 2024-11-21T13:24:36,576 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/e49f352c6e624248bb9f826ffbf12df4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e49f352c6e624248bb9f826ffbf12df4 2024-11-21T13:24:36,583 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/C of 0eb6d933356cd6a9ba9fa27a04b9b8ff into e49f352c6e624248bb9f826ffbf12df4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:36,583 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:36,583 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/C, priority=13, startTime=1732195476058; duration=0sec 2024-11-21T13:24:36,584 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:36,584 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:C 2024-11-21T13:24:36,628 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/38005dc928624fffa63f77d67c2cd28f 2024-11-21T13:24:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/68b28ce3e63a40849e42abef0e592a5e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/68b28ce3e63a40849e42abef0e592a5e 2024-11-21T13:24:36,641 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/68b28ce3e63a40849e42abef0e592a5e, entries=150, sequenceid=288, filesize=30.5 K 2024-11-21T13:24:36,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/f4f1a25c37f843899ed0d585a0d4a62c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/f4f1a25c37f843899ed0d585a0d4a62c 2024-11-21T13:24:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,648 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/f4f1a25c37f843899ed0d585a0d4a62c, entries=150, sequenceid=288, filesize=12.0 K 2024-11-21T13:24:36,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/38005dc928624fffa63f77d67c2cd28f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/38005dc928624fffa63f77d67c2cd28f 2024-11-21T13:24:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,653 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/38005dc928624fffa63f77d67c2cd28f, entries=150, sequenceid=288, filesize=12.0 K 2024-11-21T13:24:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,654 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 538ms, sequenceid=288, compaction requested=false 2024-11-21T13:24:36,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:36,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:36,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-21T13:24:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-21T13:24:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,657 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-21T13:24:36,658 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9740 sec 2024-11-21T13:24:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,660 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.9780 sec 2024-11-21T13:24:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-21T13:24:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,830 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,830 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-21T13:24:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T13:24:36,833 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,833 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:36,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:36,861 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:24:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cdd0ab1ecc0240b6b7629e25a6fa7f2b_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195476859/Put/seqid=0 2024-11-21T13:24:36,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:36,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742022_1198 (size=22618) 2024-11-21T13:24:36,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195536918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:36,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:36,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195536920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:36,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:36,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T13:24:36,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T13:24:36,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:36,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:36,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:36,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:36,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:37,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:37,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195537029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:37,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195537029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:37,032 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:37,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T13:24:37,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:37,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:37,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:37,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T13:24:37,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:37,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195537239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:37,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:37,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195537239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:37,291 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:37,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T13:24:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:37,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:37,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,302 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cdd0ab1ecc0240b6b7629e25a6fa7f2b_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cdd0ab1ecc0240b6b7629e25a6fa7f2b_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:37,306 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/360da96727694637a7fedd7f2fc7322c, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:37,307 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/360da96727694637a7fedd7f2fc7322c is 175, key is test_row_0/A:col10/1732195476859/Put/seqid=0 2024-11-21T13:24:37,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742023_1199 (size=66019) 2024-11-21T13:24:37,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T13:24:37,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:37,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:37,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:37,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T13:24:37,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195537549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:37,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:37,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195537549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:37,550 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:37,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T13:24:37,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:37,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:37,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:37,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,740 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=302, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/360da96727694637a7fedd7f2fc7322c 2024-11-21T13:24:37,749 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/63ba5c5795d1487b88d45dd546b1920d is 50, key is test_row_0/B:col10/1732195476859/Put/seqid=0 2024-11-21T13:24:37,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T13:24:37,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:37,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:37,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:37,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742024_1200 (size=9857) 2024-11-21T13:24:37,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T13:24:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:37,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T13:24:38,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T13:24:38,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:38,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:38,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:38,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:38,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:38,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:38,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:38,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195538059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:38,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:38,063 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:38,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195538059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:38,064 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:38,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/63ba5c5795d1487b88d45dd546b1920d 2024-11-21T13:24:38,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/fe8c798bde6b4822b402697eb05c1438 is 50, key is test_row_0/C:col10/1732195476859/Put/seqid=0 2024-11-21T13:24:38,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742025_1201 (size=9857) 2024-11-21T13:24:38,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T13:24:38,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:38,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:38,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:38,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:38,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:38,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:38,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T13:24:38,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:38,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:38,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:38,367 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:38,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:38,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:38,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T13:24:38,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:38,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:38,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:38,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:38,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:38,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:38,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=302 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/fe8c798bde6b4822b402697eb05c1438 2024-11-21T13:24:38,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/360da96727694637a7fedd7f2fc7322c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/360da96727694637a7fedd7f2fc7322c 2024-11-21T13:24:38,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/360da96727694637a7fedd7f2fc7322c, entries=350, sequenceid=302, filesize=64.5 K 2024-11-21T13:24:38,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/63ba5c5795d1487b88d45dd546b1920d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/63ba5c5795d1487b88d45dd546b1920d 2024-11-21T13:24:38,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/63ba5c5795d1487b88d45dd546b1920d, entries=100, sequenceid=302, filesize=9.6 K 2024-11-21T13:24:38,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/fe8c798bde6b4822b402697eb05c1438 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/fe8c798bde6b4822b402697eb05c1438 2024-11-21T13:24:38,600 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/fe8c798bde6b4822b402697eb05c1438, entries=100, sequenceid=302, filesize=9.6 K 2024-11-21T13:24:38,601 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 1740ms, sequenceid=302, compaction requested=true 2024-11-21T13:24:38,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:38,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:38,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:38,601 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:38,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:38,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:38,601 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:38,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:38,602 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:38,603 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129143 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:38,603 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:38,603 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/B is initiating minor compaction (all files) 2024-11-21T13:24:38,603 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/A is initiating minor compaction (all files) 2024-11-21T13:24:38,604 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/B in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:38,604 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/A in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:38,604 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c680e970960e451d8d51751482c37ca3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/68b28ce3e63a40849e42abef0e592a5e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/360da96727694637a7fedd7f2fc7322c] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=126.1 K 2024-11-21T13:24:38,604 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b2fa84847ec746798dc1c3dfc1ce7a0b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/f4f1a25c37f843899ed0d585a0d4a62c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/63ba5c5795d1487b88d45dd546b1920d] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=34.3 K 2024-11-21T13:24:38,604 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:38,604 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c680e970960e451d8d51751482c37ca3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/68b28ce3e63a40849e42abef0e592a5e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/360da96727694637a7fedd7f2fc7322c] 2024-11-21T13:24:38,604 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b2fa84847ec746798dc1c3dfc1ce7a0b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732195473904 2024-11-21T13:24:38,604 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting c680e970960e451d8d51751482c37ca3, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732195473904 2024-11-21T13:24:38,604 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f4f1a25c37f843899ed0d585a0d4a62c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732195474559 2024-11-21T13:24:38,605 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 63ba5c5795d1487b88d45dd546b1920d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732195476859 2024-11-21T13:24:38,605 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68b28ce3e63a40849e42abef0e592a5e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732195474559 2024-11-21T13:24:38,605 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 360da96727694637a7fedd7f2fc7322c, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732195476806 2024-11-21T13:24:38,614 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:38,615 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/e3a4793032de46c78a1c307e07b2c705 is 50, key is test_row_0/B:col10/1732195476859/Put/seqid=0 2024-11-21T13:24:38,615 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:38,618 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121430241108ba8445299658ec00aeba448_0eb6d933356cd6a9ba9fa27a04b9b8ff store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:38,620 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121430241108ba8445299658ec00aeba448_0eb6d933356cd6a9ba9fa27a04b9b8ff, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:38,621 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121430241108ba8445299658ec00aeba448_0eb6d933356cd6a9ba9fa27a04b9b8ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:38,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742026_1202 (size=13017) 2024-11-21T13:24:38,637 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/e3a4793032de46c78a1c307e07b2c705 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/e3a4793032de46c78a1c307e07b2c705 2024-11-21T13:24:38,645 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/B of 0eb6d933356cd6a9ba9fa27a04b9b8ff into e3a4793032de46c78a1c307e07b2c705(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:38,645 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:38,645 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/B, priority=13, startTime=1732195478601; duration=0sec 2024-11-21T13:24:38,645 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:38,645 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:B 2024-11-21T13:24:38,645 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:38,647 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:38,647 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/C is initiating minor compaction (all files) 2024-11-21T13:24:38,647 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/C in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:38,647 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e49f352c6e624248bb9f826ffbf12df4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/38005dc928624fffa63f77d67c2cd28f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/fe8c798bde6b4822b402697eb05c1438] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=34.3 K 2024-11-21T13:24:38,648 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e49f352c6e624248bb9f826ffbf12df4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732195473904 2024-11-21T13:24:38,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742027_1203 (size=4469) 2024-11-21T13:24:38,651 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#A#compaction#166 average throughput is 0.68 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:38,651 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/1e97d676e3374601a144ad3a68e830b3 is 175, key is test_row_0/A:col10/1732195476859/Put/seqid=0 2024-11-21T13:24:38,653 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 38005dc928624fffa63f77d67c2cd28f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1732195474559 2024-11-21T13:24:38,654 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting fe8c798bde6b4822b402697eb05c1438, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732195476859 2024-11-21T13:24:38,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742028_1204 (size=32078) 2024-11-21T13:24:38,668 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/1e97d676e3374601a144ad3a68e830b3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1e97d676e3374601a144ad3a68e830b3 2024-11-21T13:24:38,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-21T13:24:38,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:38,673 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T13:24:38,674 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/A of 0eb6d933356cd6a9ba9fa27a04b9b8ff into 1e97d676e3374601a144ad3a68e830b3(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:38,674 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:38,674 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/A, priority=13, startTime=1732195478601; duration=0sec 2024-11-21T13:24:38,674 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:38,674 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:A 2024-11-21T13:24:38,674 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#C#compaction#167 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:38,675 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/171b6ce62e98477fb5a075167ccd7447 is 50, key is test_row_0/C:col10/1732195476859/Put/seqid=0 2024-11-21T13:24:38,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d0ddb9b8d79946bdb97791393e804702_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195476918/Put/seqid=0 2024-11-21T13:24:38,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742029_1205 (size=13017) 2024-11-21T13:24:38,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742030_1206 (size=12454) 2024-11-21T13:24:38,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:38,699 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d0ddb9b8d79946bdb97791393e804702_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d0ddb9b8d79946bdb97791393e804702_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:38,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/db6de0080b26496aa3de53964a7d8670, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:38,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/db6de0080b26496aa3de53964a7d8670 is 175, key is test_row_0/A:col10/1732195476918/Put/seqid=0 2024-11-21T13:24:38,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742031_1207 (size=31255) 2024-11-21T13:24:38,706 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/db6de0080b26496aa3de53964a7d8670 2024-11-21T13:24:38,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b287491a1f6a401cbc96d1e968a09661 is 50, key is test_row_0/B:col10/1732195476918/Put/seqid=0 2024-11-21T13:24:38,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742032_1208 (size=12301) 2024-11-21T13:24:38,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T13:24:39,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:39,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:39,100 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/171b6ce62e98477fb5a075167ccd7447 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/171b6ce62e98477fb5a075167ccd7447 2024-11-21T13:24:39,108 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/C of 0eb6d933356cd6a9ba9fa27a04b9b8ff into 171b6ce62e98477fb5a075167ccd7447(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:39,108 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:39,108 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/C, priority=13, startTime=1732195478601; duration=0sec 2024-11-21T13:24:39,108 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:39,108 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:C 2024-11-21T13:24:39,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:39,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195539105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:39,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:39,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195539105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:39,131 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b287491a1f6a401cbc96d1e968a09661 2024-11-21T13:24:39,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/4ca166dd258b4627b9c5c32a064bed19 is 50, key is test_row_0/C:col10/1732195476918/Put/seqid=0 2024-11-21T13:24:39,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742033_1209 (size=12301) 2024-11-21T13:24:39,177 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/4ca166dd258b4627b9c5c32a064bed19 2024-11-21T13:24:39,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/db6de0080b26496aa3de53964a7d8670 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/db6de0080b26496aa3de53964a7d8670 2024-11-21T13:24:39,188 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/db6de0080b26496aa3de53964a7d8670, entries=150, sequenceid=329, filesize=30.5 K 2024-11-21T13:24:39,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b287491a1f6a401cbc96d1e968a09661 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b287491a1f6a401cbc96d1e968a09661 2024-11-21T13:24:39,200 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b287491a1f6a401cbc96d1e968a09661, entries=150, sequenceid=329, filesize=12.0 K 2024-11-21T13:24:39,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/4ca166dd258b4627b9c5c32a064bed19 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4ca166dd258b4627b9c5c32a064bed19 2024-11-21T13:24:39,207 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4ca166dd258b4627b9c5c32a064bed19, entries=150, sequenceid=329, filesize=12.0 K 2024-11-21T13:24:39,208 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 534ms, sequenceid=329, compaction requested=false 2024-11-21T13:24:39,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:39,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:39,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-21T13:24:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-21T13:24:39,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-21T13:24:39,212 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3760 sec 2024-11-21T13:24:39,217 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.3820 sec 2024-11-21T13:24:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:39,221 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:39,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T13:24:39,241 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121fc204dc8e9734f39ad7288afe42a6748_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195479083/Put/seqid=0 2024-11-21T13:24:39,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742034_1210 (size=14994) 2024-11-21T13:24:39,254 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:39,259 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121fc204dc8e9734f39ad7288afe42a6748_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fc204dc8e9734f39ad7288afe42a6748_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:39,260 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/5b6033ef3eac4e5bacd8e19d03c75a31, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:39,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/5b6033ef3eac4e5bacd8e19d03c75a31 is 175, key is test_row_0/A:col10/1732195479083/Put/seqid=0 2024-11-21T13:24:39,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742035_1211 (size=39949) 2024-11-21T13:24:39,299 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/5b6033ef3eac4e5bacd8e19d03c75a31 2024-11-21T13:24:39,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b1825bd09a764513baab0f269dfff8ea is 50, key is test_row_0/B:col10/1732195479083/Put/seqid=0 2024-11-21T13:24:39,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195539309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,314 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:39,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195539313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:39,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742036_1212 (size=12301) 2024-11-21T13:24:39,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195539420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:39,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195539425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:39,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195539629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:39,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195539629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,630 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:39,739 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b1825bd09a764513baab0f269dfff8ea 2024-11-21T13:24:39,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/9ecebb3a697c4f41b8f6d298c856f4c8 is 50, key is test_row_0/C:col10/1732195479083/Put/seqid=0 2024-11-21T13:24:39,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742037_1213 (size=12301) 2024-11-21T13:24:39,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/9ecebb3a697c4f41b8f6d298c856f4c8 2024-11-21T13:24:39,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/5b6033ef3eac4e5bacd8e19d03c75a31 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/5b6033ef3eac4e5bacd8e19d03c75a31 2024-11-21T13:24:39,811 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/5b6033ef3eac4e5bacd8e19d03c75a31, entries=200, sequenceid=342, filesize=39.0 K 2024-11-21T13:24:39,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b1825bd09a764513baab0f269dfff8ea as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b1825bd09a764513baab0f269dfff8ea 2024-11-21T13:24:39,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b1825bd09a764513baab0f269dfff8ea, entries=150, sequenceid=342, filesize=12.0 K 2024-11-21T13:24:39,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/9ecebb3a697c4f41b8f6d298c856f4c8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/9ecebb3a697c4f41b8f6d298c856f4c8 2024-11-21T13:24:39,828 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/9ecebb3a697c4f41b8f6d298c856f4c8, entries=150, sequenceid=342, filesize=12.0 K 2024-11-21T13:24:39,829 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-21T13:24:39,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 612ms, sequenceid=342, compaction requested=true 2024-11-21T13:24:39,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:39,833 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:39,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:39,834 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103282 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:39,834 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/A is initiating minor compaction (all files) 2024-11-21T13:24:39,834 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/A in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:39,835 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1e97d676e3374601a144ad3a68e830b3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/db6de0080b26496aa3de53964a7d8670, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/5b6033ef3eac4e5bacd8e19d03c75a31] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=100.9 K 2024-11-21T13:24:39,835 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:39,835 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1e97d676e3374601a144ad3a68e830b3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/db6de0080b26496aa3de53964a7d8670, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/5b6033ef3eac4e5bacd8e19d03c75a31] 2024-11-21T13:24:39,835 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1e97d676e3374601a144ad3a68e830b3, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732195474566 2024-11-21T13:24:39,835 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting db6de0080b26496aa3de53964a7d8670, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732195476903 2024-11-21T13:24:39,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:39,836 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b6033ef3eac4e5bacd8e19d03c75a31, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732195479083 2024-11-21T13:24:39,837 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:39,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:39,838 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:39,839 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:39,839 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/B is initiating minor compaction (all files) 2024-11-21T13:24:39,839 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/B in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:39,839 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/e3a4793032de46c78a1c307e07b2c705, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b287491a1f6a401cbc96d1e968a09661, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b1825bd09a764513baab0f269dfff8ea] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=36.7 K 2024-11-21T13:24:39,840 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e3a4793032de46c78a1c307e07b2c705, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732195474566 2024-11-21T13:24:39,841 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b287491a1f6a401cbc96d1e968a09661, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732195476903 2024-11-21T13:24:39,841 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b1825bd09a764513baab0f269dfff8ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732195479083 2024-11-21T13:24:39,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:39,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:39,851 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:39,874 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#B#compaction#175 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:39,876 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b46b2bf1a8984b4dbae0eff91dddab24 is 50, key is test_row_0/B:col10/1732195479083/Put/seqid=0 2024-11-21T13:24:39,886 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112149ccf76f59b64489a4e4f6fc63796a4a_0eb6d933356cd6a9ba9fa27a04b9b8ff store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:39,889 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112149ccf76f59b64489a4e4f6fc63796a4a_0eb6d933356cd6a9ba9fa27a04b9b8ff, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:39,889 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112149ccf76f59b64489a4e4f6fc63796a4a_0eb6d933356cd6a9ba9fa27a04b9b8ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:39,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742038_1214 (size=13119) 2024-11-21T13:24:39,937 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b46b2bf1a8984b4dbae0eff91dddab24 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b46b2bf1a8984b4dbae0eff91dddab24 2024-11-21T13:24:39,944 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/B of 0eb6d933356cd6a9ba9fa27a04b9b8ff into b46b2bf1a8984b4dbae0eff91dddab24(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:39,945 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:39,945 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/B, priority=13, startTime=1732195479836; duration=0sec 2024-11-21T13:24:39,945 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:39,945 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:B 2024-11-21T13:24:39,945 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:39,946 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:39,946 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T13:24:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:39,949 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:39,949 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/C is initiating minor compaction (all files) 2024-11-21T13:24:39,949 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/C in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:39,949 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/171b6ce62e98477fb5a075167ccd7447, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4ca166dd258b4627b9c5c32a064bed19, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/9ecebb3a697c4f41b8f6d298c856f4c8] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=36.7 K 2024-11-21T13:24:39,949 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 171b6ce62e98477fb5a075167ccd7447, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=302, earliestPutTs=1732195474566 2024-11-21T13:24:39,950 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ca166dd258b4627b9c5c32a064bed19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732195476903 2024-11-21T13:24:39,951 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ecebb3a697c4f41b8f6d298c856f4c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732195479083 2024-11-21T13:24:39,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742039_1215 (size=4469) 2024-11-21T13:24:39,955 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#A#compaction#174 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:39,955 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/ab2b5e107e064d0185cd467beabc9882 is 175, key is test_row_0/A:col10/1732195479083/Put/seqid=0 2024-11-21T13:24:39,960 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112141ae4793ee1a4d779d23cf66bdc71ace_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195479272/Put/seqid=0 2024-11-21T13:24:39,971 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#C#compaction#177 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:39,972 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/ca3f204cd2fc4bc091cceac06ce47d58 is 50, key is test_row_0/C:col10/1732195479083/Put/seqid=0 2024-11-21T13:24:39,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195539974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:39,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742040_1216 (size=32073) 2024-11-21T13:24:39,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195539980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:39,996 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/ab2b5e107e064d0185cd467beabc9882 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/ab2b5e107e064d0185cd467beabc9882 2024-11-21T13:24:40,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742041_1217 (size=14994) 2024-11-21T13:24:40,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742042_1218 (size=13119) 2024-11-21T13:24:40,034 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/ca3f204cd2fc4bc091cceac06ce47d58 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/ca3f204cd2fc4bc091cceac06ce47d58 2024-11-21T13:24:40,039 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/A of 0eb6d933356cd6a9ba9fa27a04b9b8ff into ab2b5e107e064d0185cd467beabc9882(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:40,039 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:40,039 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/A, priority=13, startTime=1732195479833; duration=0sec 2024-11-21T13:24:40,039 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:40,039 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:A 2024-11-21T13:24:40,042 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/C of 0eb6d933356cd6a9ba9fa27a04b9b8ff into ca3f204cd2fc4bc091cceac06ce47d58(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:40,042 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:40,042 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/C, priority=13, startTime=1732195479838; duration=0sec 2024-11-21T13:24:40,042 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:40,042 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:C 2024-11-21T13:24:40,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195540089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:40,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195540089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:40,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:40,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:40,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195540299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:40,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:40,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195540300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:40,304 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:40,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:40,426 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:40,433 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112141ae4793ee1a4d779d23cf66bdc71ace_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112141ae4793ee1a4d779d23cf66bdc71ace_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:40,435 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/1b4e08cb9e6b4be389b975636216fe92, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:40,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/1b4e08cb9e6b4be389b975636216fe92 is 175, key is test_row_0/A:col10/1732195479272/Put/seqid=0 2024-11-21T13:24:40,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742043_1219 (size=39949) 2024-11-21T13:24:40,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:40,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195540609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:40,610 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:40,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195540609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:40,958 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=368, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/1b4e08cb9e6b4be389b975636216fe92 2024-11-21T13:24:40,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/86d2980284fa44ccad19c9dab18b040f is 50, key is test_row_0/B:col10/1732195479272/Put/seqid=0 2024-11-21T13:24:41,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-21T13:24:41,010 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:41,011 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-21T13:24:41,014 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:41,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T13:24:41,016 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:41,017 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:41,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742044_1220 (size=12301) 2024-11-21T13:24:41,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T13:24:41,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:41,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195541120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:41,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:41,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:41,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195541120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:41,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:41,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:41,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:41,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:41,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:41,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T13:24:41,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:41,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:41,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:41,170 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:41,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:41,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:41,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T13:24:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:41,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:41,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:41,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:41,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T13:24:41,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/86d2980284fa44ccad19c9dab18b040f 2024-11-21T13:24:41,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/e94014bb7aa44ebc973f10bcbb6455ab is 50, key is test_row_0/C:col10/1732195479272/Put/seqid=0 2024-11-21T13:24:41,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742045_1221 (size=12301) 2024-11-21T13:24:41,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=368 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/e94014bb7aa44ebc973f10bcbb6455ab 2024-11-21T13:24:41,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/1b4e08cb9e6b4be389b975636216fe92 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1b4e08cb9e6b4be389b975636216fe92 2024-11-21T13:24:41,477 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T13:24:41,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:41,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:41,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:41,477 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:41,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:41,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:41,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1b4e08cb9e6b4be389b975636216fe92, entries=200, sequenceid=368, filesize=39.0 K 2024-11-21T13:24:41,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/86d2980284fa44ccad19c9dab18b040f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/86d2980284fa44ccad19c9dab18b040f 2024-11-21T13:24:41,493 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/86d2980284fa44ccad19c9dab18b040f, entries=150, sequenceid=368, filesize=12.0 K 2024-11-21T13:24:41,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/e94014bb7aa44ebc973f10bcbb6455ab as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e94014bb7aa44ebc973f10bcbb6455ab 2024-11-21T13:24:41,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e94014bb7aa44ebc973f10bcbb6455ab, entries=150, sequenceid=368, filesize=12.0 K 2024-11-21T13:24:41,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 1563ms, sequenceid=368, compaction requested=false 2024-11-21T13:24:41,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:41,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-21T13:24:41,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:41,632 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:24:41,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T13:24:41,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b3105ca5c1234973b6afaaa5c89138de_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195479978/Put/seqid=0 2024-11-21T13:24:41,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742046_1222 (size=12454) 2024-11-21T13:24:41,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:41,662 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b3105ca5c1234973b6afaaa5c89138de_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b3105ca5c1234973b6afaaa5c89138de_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:41,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/da51c61c692b4f1cbb1899da082894a7, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:41,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/da51c61c692b4f1cbb1899da082894a7 is 175, key is test_row_0/A:col10/1732195479978/Put/seqid=0 2024-11-21T13:24:41,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742047_1223 (size=31255) 2024-11-21T13:24:41,673 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=381, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/da51c61c692b4f1cbb1899da082894a7 2024-11-21T13:24:41,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/64e53110ce3e497bafaf49b66a725a12 is 50, key is test_row_0/B:col10/1732195479978/Put/seqid=0 2024-11-21T13:24:41,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742048_1224 (size=12301) 2024-11-21T13:24:42,094 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/64e53110ce3e497bafaf49b66a725a12 2024-11-21T13:24:42,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/24440117f333478cb1548d3565914011 is 50, key is test_row_0/C:col10/1732195479978/Put/seqid=0 2024-11-21T13:24:42,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742049_1225 (size=12301) 2024-11-21T13:24:42,140 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/24440117f333478cb1548d3565914011 2024-11-21T13:24:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:42,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:42,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/da51c61c692b4f1cbb1899da082894a7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/da51c61c692b4f1cbb1899da082894a7 2024-11-21T13:24:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T13:24:42,154 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/da51c61c692b4f1cbb1899da082894a7, entries=150, sequenceid=381, filesize=30.5 K 2024-11-21T13:24:42,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/64e53110ce3e497bafaf49b66a725a12 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/64e53110ce3e497bafaf49b66a725a12 2024-11-21T13:24:42,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,159 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/64e53110ce3e497bafaf49b66a725a12, entries=150, sequenceid=381, filesize=12.0 K 2024-11-21T13:24:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/24440117f333478cb1548d3565914011 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/24440117f333478cb1548d3565914011 2024-11-21T13:24:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,167 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/24440117f333478cb1548d3565914011, entries=150, sequenceid=381, filesize=12.0 K 2024-11-21T13:24:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,168 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=73.80 KB/75570 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 536ms, sequenceid=381, compaction requested=true 2024-11-21T13:24:42,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:42,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:42,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-21T13:24:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-21T13:24:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-21T13:24:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,172 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1520 sec 2024-11-21T13:24:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.1620 sec 2024-11-21T13:24:42,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:42,194 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:42,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T13:24:42,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121914105d0802e4008bc64a4ee7ade6d10_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_1/A:col10/1732195482194/Put/seqid=0 2024-11-21T13:24:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742050_1226 (size=20074) 2024-11-21T13:24:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,236 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,242 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121914105d0802e4008bc64a4ee7ade6d10_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121914105d0802e4008bc64a4ee7ade6d10_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,243 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/fdcf755c53d1431e8295d48d4ae2f71d, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/fdcf755c53d1431e8295d48d4ae2f71d is 175, key is test_row_1/A:col10/1732195482194/Put/seqid=0 2024-11-21T13:24:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:42,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742051_1227 (size=57329) 2024-11-21T13:24:42,266 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=397, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/fdcf755c53d1431e8295d48d4ae2f71d 2024-11-21T13:24:42,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/26024d6270f34d6da3e49797cdafb475 is 50, key is test_row_1/B:col10/1732195482194/Put/seqid=0 2024-11-21T13:24:42,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742052_1228 (size=9857) 2024-11-21T13:24:42,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/26024d6270f34d6da3e49797cdafb475 2024-11-21T13:24:42,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/31a1fc8a959546fa80ef87c72429e0e4 is 50, key is test_row_1/C:col10/1732195482194/Put/seqid=0 2024-11-21T13:24:42,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742053_1229 (size=9857) 2024-11-21T13:24:42,315 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:42,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195542311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:42,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195542311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,317 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:42,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,318 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:42,424 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195542420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:42,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195542420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:42,425 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:42,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:42,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195542629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:42,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195542630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:42,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:42,715 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=397 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/31a1fc8a959546fa80ef87c72429e0e4 2024-11-21T13:24:42,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/fdcf755c53d1431e8295d48d4ae2f71d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/fdcf755c53d1431e8295d48d4ae2f71d 2024-11-21T13:24:42,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/fdcf755c53d1431e8295d48d4ae2f71d, entries=300, sequenceid=397, filesize=56.0 K 2024-11-21T13:24:42,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/26024d6270f34d6da3e49797cdafb475 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/26024d6270f34d6da3e49797cdafb475 2024-11-21T13:24:42,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/26024d6270f34d6da3e49797cdafb475, entries=100, sequenceid=397, filesize=9.6 K 2024-11-21T13:24:42,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/31a1fc8a959546fa80ef87c72429e0e4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/31a1fc8a959546fa80ef87c72429e0e4 2024-11-21T13:24:42,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/31a1fc8a959546fa80ef87c72429e0e4, entries=100, sequenceid=397, filesize=9.6 K 2024-11-21T13:24:42,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 550ms, sequenceid=397, compaction requested=true 2024-11-21T13:24:42,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:42,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:42,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:42,745 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:42,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:42,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:42,745 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:42,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:42,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:42,746 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 160606 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:42,746 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/A is initiating minor compaction (all files) 2024-11-21T13:24:42,746 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47578 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:42,746 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/A in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:42,746 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/B is initiating minor compaction (all files) 2024-11-21T13:24:42,746 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/B in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:42,747 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/ab2b5e107e064d0185cd467beabc9882, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1b4e08cb9e6b4be389b975636216fe92, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/da51c61c692b4f1cbb1899da082894a7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/fdcf755c53d1431e8295d48d4ae2f71d] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=156.8 K 2024-11-21T13:24:42,747 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b46b2bf1a8984b4dbae0eff91dddab24, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/86d2980284fa44ccad19c9dab18b040f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/64e53110ce3e497bafaf49b66a725a12, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/26024d6270f34d6da3e49797cdafb475] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=46.5 K 2024-11-21T13:24:42,747 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:42,747 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/ab2b5e107e064d0185cd467beabc9882, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1b4e08cb9e6b4be389b975636216fe92, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/da51c61c692b4f1cbb1899da082894a7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/fdcf755c53d1431e8295d48d4ae2f71d] 2024-11-21T13:24:42,747 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b46b2bf1a8984b4dbae0eff91dddab24, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732195479083 2024-11-21T13:24:42,747 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab2b5e107e064d0185cd467beabc9882, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732195479083 2024-11-21T13:24:42,747 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 86d2980284fa44ccad19c9dab18b040f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732195479272 2024-11-21T13:24:42,747 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b4e08cb9e6b4be389b975636216fe92, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732195479272 2024-11-21T13:24:42,748 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 64e53110ce3e497bafaf49b66a725a12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732195479965 2024-11-21T13:24:42,748 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting da51c61c692b4f1cbb1899da082894a7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732195479965 2024-11-21T13:24:42,748 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 26024d6270f34d6da3e49797cdafb475, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732195482163 2024-11-21T13:24:42,748 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdcf755c53d1431e8295d48d4ae2f71d, keycount=300, bloomtype=ROW, size=56.0 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732195482150 2024-11-21T13:24:42,770 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:42,787 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#B#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:42,788 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/bd5f5555e093429f90dd8b90eaa66a53 is 50, key is test_row_0/B:col10/1732195479978/Put/seqid=0 2024-11-21T13:24:42,793 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411210e4a764872844544b3ff7ce054470f37_0eb6d933356cd6a9ba9fa27a04b9b8ff store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:42,795 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411210e4a764872844544b3ff7ce054470f37_0eb6d933356cd6a9ba9fa27a04b9b8ff, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:42,795 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210e4a764872844544b3ff7ce054470f37_0eb6d933356cd6a9ba9fa27a04b9b8ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:42,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742054_1230 (size=13255) 2024-11-21T13:24:42,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742055_1231 (size=4469) 2024-11-21T13:24:42,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:42,942 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:42,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T13:24:42,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:42,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195542972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:42,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210dfc9c2ad51c405483dce0077c40d01a_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195482304/Put/seqid=0 2024-11-21T13:24:42,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:42,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195542975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:42,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:42,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742056_1232 (size=14994) 2024-11-21T13:24:42,992 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:43,002 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210dfc9c2ad51c405483dce0077c40d01a_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210dfc9c2ad51c405483dce0077c40d01a_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:43,004 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/dc150fd36a8e47f0b4829d4a2dd90085, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:43,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/dc150fd36a8e47f0b4829d4a2dd90085 is 175, key is test_row_0/A:col10/1732195482304/Put/seqid=0 2024-11-21T13:24:43,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742057_1233 (size=39949) 2024-11-21T13:24:43,011 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=419, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/dc150fd36a8e47f0b4829d4a2dd90085 2024-11-21T13:24:43,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/176fe43c31de480c8654ab97d022349a is 50, key is test_row_0/B:col10/1732195482304/Put/seqid=0 2024-11-21T13:24:43,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742058_1234 (size=12301) 2024-11-21T13:24:43,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/176fe43c31de480c8654ab97d022349a 2024-11-21T13:24:43,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/957af8775242408f9a4c44c45ba1df1b is 50, key is test_row_0/C:col10/1732195482304/Put/seqid=0 2024-11-21T13:24:43,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742059_1235 (size=12301) 2024-11-21T13:24:43,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/957af8775242408f9a4c44c45ba1df1b 2024-11-21T13:24:43,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:43,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195543079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,090 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:43,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/dc150fd36a8e47f0b4829d4a2dd90085 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/dc150fd36a8e47f0b4829d4a2dd90085 2024-11-21T13:24:43,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:43,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195543089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,092 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:43,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/dc150fd36a8e47f0b4829d4a2dd90085, entries=200, sequenceid=419, filesize=39.0 K 2024-11-21T13:24:43,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/176fe43c31de480c8654ab97d022349a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/176fe43c31de480c8654ab97d022349a 2024-11-21T13:24:43,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/176fe43c31de480c8654ab97d022349a, entries=150, sequenceid=419, filesize=12.0 K 2024-11-21T13:24:43,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/957af8775242408f9a4c44c45ba1df1b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/957af8775242408f9a4c44c45ba1df1b 2024-11-21T13:24:43,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/957af8775242408f9a4c44c45ba1df1b, entries=150, sequenceid=419, filesize=12.0 K 2024-11-21T13:24:43,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 172ms, sequenceid=419, compaction requested=true 2024-11-21T13:24:43,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:43,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:A, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:43,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T13:24:43,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:B, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:43,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-21T13:24:43,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:43,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-21T13:24:43,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-21T13:24:43,159 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:43,160 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-21T13:24:43,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T13:24:43,162 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:43,163 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:43,163 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:43,204 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#A#compaction#186 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:43,205 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/e7c9c321f7e6476e872fae347071f9f9 is 175, key is test_row_0/A:col10/1732195479978/Put/seqid=0 2024-11-21T13:24:43,209 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/bd5f5555e093429f90dd8b90eaa66a53 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/bd5f5555e093429f90dd8b90eaa66a53 2024-11-21T13:24:43,222 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/B of 0eb6d933356cd6a9ba9fa27a04b9b8ff into bd5f5555e093429f90dd8b90eaa66a53(size=12.9 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:43,222 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:43,222 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/B, priority=12, startTime=1732195482745; duration=0sec 2024-11-21T13:24:43,224 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-21T13:24:43,224 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:B 2024-11-21T13:24:43,224 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-21T13:24:43,232 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T13:24:43,233 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T13:24:43,233 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. because compaction request was cancelled 2024-11-21T13:24:43,235 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:A 2024-11-21T13:24:43,235 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-21T13:24:43,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742060_1236 (size=32316) 2024-11-21T13:24:43,237 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59879 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-21T13:24:43,237 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/C is initiating minor compaction (all files) 2024-11-21T13:24:43,237 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/C in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:43,237 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/ca3f204cd2fc4bc091cceac06ce47d58, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e94014bb7aa44ebc973f10bcbb6455ab, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/24440117f333478cb1548d3565914011, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/31a1fc8a959546fa80ef87c72429e0e4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/957af8775242408f9a4c44c45ba1df1b] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=58.5 K 2024-11-21T13:24:43,238 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting ca3f204cd2fc4bc091cceac06ce47d58, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1732195479083 2024-11-21T13:24:43,238 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e94014bb7aa44ebc973f10bcbb6455ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=368, earliestPutTs=1732195479272 2024-11-21T13:24:43,238 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 24440117f333478cb1548d3565914011, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1732195479965 2024-11-21T13:24:43,239 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 31a1fc8a959546fa80ef87c72429e0e4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732195482163 2024-11-21T13:24:43,239 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 957af8775242408f9a4c44c45ba1df1b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732195482304 2024-11-21T13:24:43,249 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/e7c9c321f7e6476e872fae347071f9f9 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e7c9c321f7e6476e872fae347071f9f9 2024-11-21T13:24:43,259 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/A of 0eb6d933356cd6a9ba9fa27a04b9b8ff into e7c9c321f7e6476e872fae347071f9f9(size=31.6 K), total size for store is 70.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:43,259 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:43,259 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/A, priority=12, startTime=1732195482745; duration=0sec 2024-11-21T13:24:43,259 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T13:24:43,259 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:A 2024-11-21T13:24:43,259 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:B 2024-11-21T13:24:43,259 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 5 compacting, 0 eligible, 16 blocking 2024-11-21T13:24:43,259 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T13:24:43,259 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T13:24:43,259 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. because compaction request was cancelled 2024-11-21T13:24:43,259 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:C 2024-11-21T13:24:43,259 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T13:24:43,260 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T13:24:43,260 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T13:24:43,260 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. because compaction request was cancelled 2024-11-21T13:24:43,260 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:B 2024-11-21T13:24:43,263 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#C#compaction#191 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:43,264 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/e5a2c24fa5f74262bbc91055e56dbcdb is 50, key is test_row_0/C:col10/1732195482304/Put/seqid=0 2024-11-21T13:24:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T13:24:43,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742061_1237 (size=13289) 2024-11-21T13:24:43,280 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/e5a2c24fa5f74262bbc91055e56dbcdb as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e5a2c24fa5f74262bbc91055e56dbcdb 2024-11-21T13:24:43,287 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/C of 0eb6d933356cd6a9ba9fa27a04b9b8ff into e5a2c24fa5f74262bbc91055e56dbcdb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:43,287 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:43,287 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/C, priority=11, startTime=1732195483114; duration=0sec 2024-11-21T13:24:43,288 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:43,288 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:C 2024-11-21T13:24:43,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:43,301 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:43,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T13:24:43,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121970fac9cf5d04409a47238e6c74fc07e_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195482970/Put/seqid=0 2024-11-21T13:24:43,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742062_1238 (size=12454) 2024-11-21T13:24:43,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T13:24:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:43,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:43,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 221 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195543347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:43,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195543348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:43,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:43,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195543459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:43,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:43,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195543459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:43,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T13:24:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:43,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T13:24:43,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T13:24:43,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:43,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:43,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:43,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:43,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 215 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195543669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:43,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195543669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:43,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:43,716 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:43,721 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121970fac9cf5d04409a47238e6c74fc07e_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121970fac9cf5d04409a47238e6c74fc07e_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:43,722 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/e5cfb3cdab404627881c42e34eac185d, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:43,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/e5cfb3cdab404627881c42e34eac185d is 175, key is test_row_0/A:col10/1732195482970/Put/seqid=0 2024-11-21T13:24:43,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742063_1239 (size=31255) 2024-11-21T13:24:43,730 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=438, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/e5cfb3cdab404627881c42e34eac185d 2024-11-21T13:24:43,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/49f792811e354347bbd63302b88e840f is 50, key is test_row_0/B:col10/1732195482970/Put/seqid=0 2024-11-21T13:24:43,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742064_1240 (size=12301) 2024-11-21T13:24:43,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T13:24:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:43,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T13:24:43,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T13:24:43,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:43,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:43,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:43,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:43,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195543979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:43,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:43,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195543979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:43,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:44,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T13:24:44,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:44,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:44,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:44,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:44,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:44,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:44,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/49f792811e354347bbd63302b88e840f 2024-11-21T13:24:44,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/fd0cf7ca909d4840a5bedce23a60ced0 is 50, key is test_row_0/C:col10/1732195482970/Put/seqid=0 2024-11-21T13:24:44,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742065_1241 (size=12301) 2024-11-21T13:24:44,204 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/fd0cf7ca909d4840a5bedce23a60ced0 2024-11-21T13:24:44,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/e5cfb3cdab404627881c42e34eac185d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e5cfb3cdab404627881c42e34eac185d 2024-11-21T13:24:44,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e5cfb3cdab404627881c42e34eac185d, entries=150, sequenceid=438, filesize=30.5 K 2024-11-21T13:24:44,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/49f792811e354347bbd63302b88e840f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/49f792811e354347bbd63302b88e840f 2024-11-21T13:24:44,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/49f792811e354347bbd63302b88e840f, entries=150, sequenceid=438, filesize=12.0 K 2024-11-21T13:24:44,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/fd0cf7ca909d4840a5bedce23a60ced0 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/fd0cf7ca909d4840a5bedce23a60ced0 2024-11-21T13:24:44,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/fd0cf7ca909d4840a5bedce23a60ced0, entries=150, sequenceid=438, filesize=12.0 K 2024-11-21T13:24:44,227 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 926ms, sequenceid=438, compaction requested=true 2024-11-21T13:24:44,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:44,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:44,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:44,228 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:44,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:44,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:44,228 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:44,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:44,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:44,228 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:44,228 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103520 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:44,228 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/B is initiating minor compaction (all files) 2024-11-21T13:24:44,228 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/A is initiating minor compaction (all files) 2024-11-21T13:24:44,229 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/A in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:44,229 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/B in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:44,229 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e7c9c321f7e6476e872fae347071f9f9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/dc150fd36a8e47f0b4829d4a2dd90085, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e5cfb3cdab404627881c42e34eac185d] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=101.1 K 2024-11-21T13:24:44,229 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/bd5f5555e093429f90dd8b90eaa66a53, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/176fe43c31de480c8654ab97d022349a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/49f792811e354347bbd63302b88e840f] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=37.0 K 2024-11-21T13:24:44,229 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:44,229 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e7c9c321f7e6476e872fae347071f9f9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/dc150fd36a8e47f0b4829d4a2dd90085, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e5cfb3cdab404627881c42e34eac185d] 2024-11-21T13:24:44,229 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting bd5f5555e093429f90dd8b90eaa66a53, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732195479978 2024-11-21T13:24:44,229 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7c9c321f7e6476e872fae347071f9f9, keycount=150, bloomtype=ROW, size=31.6 K, encoding=NONE, compression=NONE, seqNum=397, earliestPutTs=1732195479978 2024-11-21T13:24:44,229 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 176fe43c31de480c8654ab97d022349a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732195482304 2024-11-21T13:24:44,229 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc150fd36a8e47f0b4829d4a2dd90085, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732195482304 2024-11-21T13:24:44,229 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 49f792811e354347bbd63302b88e840f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732195482970 2024-11-21T13:24:44,230 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5cfb3cdab404627881c42e34eac185d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732195482970 2024-11-21T13:24:44,239 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:44,239 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#B#compaction#195 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:44,240 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/51be5eb3f5d94729811385f19cc30da4 is 50, key is test_row_0/B:col10/1732195482970/Put/seqid=0 2024-11-21T13:24:44,243 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411214cff16203705489ca08bbcbf2027a214_0eb6d933356cd6a9ba9fa27a04b9b8ff store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:44,245 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411214cff16203705489ca08bbcbf2027a214_0eb6d933356cd6a9ba9fa27a04b9b8ff, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:44,246 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214cff16203705489ca08bbcbf2027a214_0eb6d933356cd6a9ba9fa27a04b9b8ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:44,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-21T13:24:44,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:44,247 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T13:24:44,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742066_1242 (size=13357) 2024-11-21T13:24:44,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742067_1243 (size=4469) 2024-11-21T13:24:44,256 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#A#compaction#196 average throughput is 1.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:44,256 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/b1e1612f8c97493284f2d284639c2fcf is 175, key is test_row_0/A:col10/1732195482970/Put/seqid=0 2024-11-21T13:24:44,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217e0b1a64db214c08a472b5161354366d_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195483333/Put/seqid=0 2024-11-21T13:24:44,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742068_1244 (size=32311) 2024-11-21T13:24:44,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742069_1245 (size=12454) 2024-11-21T13:24:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T13:24:44,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:44,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:44,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:44,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195544516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:44,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:44,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 232 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195544522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:44,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:44,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:44,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195544629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:44,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 233 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195544629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:44,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:44,654 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/51be5eb3f5d94729811385f19cc30da4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/51be5eb3f5d94729811385f19cc30da4 2024-11-21T13:24:44,659 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/B of 0eb6d933356cd6a9ba9fa27a04b9b8ff into 51be5eb3f5d94729811385f19cc30da4(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:44,659 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:44,659 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/B, priority=13, startTime=1732195484228; duration=0sec 2024-11-21T13:24:44,659 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:44,659 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:B 2024-11-21T13:24:44,659 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-21T13:24:44,660 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-21T13:24:44,660 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-21T13:24:44,660 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. because compaction request was cancelled 2024-11-21T13:24:44,660 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:C 2024-11-21T13:24:44,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:44,668 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/b1e1612f8c97493284f2d284639c2fcf as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/b1e1612f8c97493284f2d284639c2fcf 2024-11-21T13:24:44,669 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217e0b1a64db214c08a472b5161354366d_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217e0b1a64db214c08a472b5161354366d_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:44,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/8418dc2e242d437783e730307f5bd2a3, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:44,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/8418dc2e242d437783e730307f5bd2a3 is 175, key is test_row_0/A:col10/1732195483333/Put/seqid=0 2024-11-21T13:24:44,672 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/A of 0eb6d933356cd6a9ba9fa27a04b9b8ff into b1e1612f8c97493284f2d284639c2fcf(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:44,672 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:44,672 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/A, priority=13, startTime=1732195484227; duration=0sec 2024-11-21T13:24:44,672 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:44,672 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:A 2024-11-21T13:24:44,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742070_1246 (size=31255) 2024-11-21T13:24:44,675 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=458, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/8418dc2e242d437783e730307f5bd2a3 2024-11-21T13:24:44,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/7930034c1c444b379750f71dab0188ac is 50, key is test_row_0/B:col10/1732195483333/Put/seqid=0 2024-11-21T13:24:44,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742071_1247 (size=12301) 2024-11-21T13:24:44,700 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/7930034c1c444b379750f71dab0188ac 2024-11-21T13:24:44,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/d584a5ae4361448080abb974f7b6e3f0 is 50, key is test_row_0/C:col10/1732195483333/Put/seqid=0 2024-11-21T13:24:44,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742072_1248 (size=12301) 2024-11-21T13:24:44,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:44,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195544839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:44,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:44,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195544839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:44,841 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:45,111 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/d584a5ae4361448080abb974f7b6e3f0 2024-11-21T13:24:45,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/8418dc2e242d437783e730307f5bd2a3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/8418dc2e242d437783e730307f5bd2a3 2024-11-21T13:24:45,119 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/8418dc2e242d437783e730307f5bd2a3, entries=150, sequenceid=458, filesize=30.5 K 2024-11-21T13:24:45,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/7930034c1c444b379750f71dab0188ac as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/7930034c1c444b379750f71dab0188ac 2024-11-21T13:24:45,124 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/7930034c1c444b379750f71dab0188ac, entries=150, sequenceid=458, filesize=12.0 K 2024-11-21T13:24:45,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/d584a5ae4361448080abb974f7b6e3f0 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/d584a5ae4361448080abb974f7b6e3f0 2024-11-21T13:24:45,130 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/d584a5ae4361448080abb974f7b6e3f0, entries=150, sequenceid=458, filesize=12.0 K 2024-11-21T13:24:45,131 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 884ms, sequenceid=458, compaction requested=true 2024-11-21T13:24:45,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:45,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:45,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-21T13:24:45,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-21T13:24:45,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-21T13:24:45,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9690 sec 2024-11-21T13:24:45,136 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.9750 sec 2024-11-21T13:24:45,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:45,152 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:24:45,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T13:24:45,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212ead4d37d3ac4fb1b4b3ddcd86766c9a_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195484520/Put/seqid=0 2024-11-21T13:24:45,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742073_1249 (size=17534) 2024-11-21T13:24:45,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:45,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 234 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195545185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:45,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:45,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195545186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:45,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:45,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195545299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:45,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:45,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195545299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,303 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:45,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-21T13:24:45,319 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:45,319 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:24:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-21T13:24:45,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T13:24:45,321 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:24:45,321 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:24:45,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:24:45,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T13:24:45,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T13:24:45,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:45,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:45,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:45,474 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:45,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:45,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:45,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:45,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195545510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:45,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,512 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:45,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 246 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195545512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:45,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:45,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55828 deadline: 1732195545539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:45,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:45,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55854 deadline: 1732195545539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:45,543 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 0eb6d933356cd6a9ba9fa27a04b9b8ff of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18298 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:45,544 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 0eb6d933356cd6a9ba9fa27a04b9b8ff of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18301 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:45,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:45,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55838 deadline: 1732195545549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:45,550 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_1' in 0eb6d933356cd6a9ba9fa27a04b9b8ff of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18307 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:24:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:45,570 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:45,573 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212ead4d37d3ac4fb1b4b3ddcd86766c9a_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212ead4d37d3ac4fb1b4b3ddcd86766c9a_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:45,577 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/6c9b92ac225a456f81c04ca93ea895e3, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:45,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/6c9b92ac225a456f81c04ca93ea895e3 is 175, key is test_row_0/A:col10/1732195484520/Put/seqid=0 2024-11-21T13:24:45,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742074_1250 (size=48639) 2024-11-21T13:24:45,583 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=478, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/6c9b92ac225a456f81c04ca93ea895e3 2024-11-21T13:24:45,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b0247b1ba1e74c36b7a036f3c9f0198a is 50, key is test_row_0/B:col10/1732195484520/Put/seqid=0 2024-11-21T13:24:45,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742075_1251 (size=12301) 2024-11-21T13:24:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T13:24:45,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T13:24:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:45,644 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:45,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:45,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T13:24:45,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:45,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:45,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:45,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:45,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:45,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55790 deadline: 1732195545819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:45,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:24:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 247 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55788 deadline: 1732195545820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:24:45,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:24:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T13:24:45,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T13:24:45,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:45,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:45,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:45,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:45,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:45,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:24:46,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b0247b1ba1e74c36b7a036f3c9f0198a 2024-11-21T13:24:46,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/4518dfb99d154be9b82a9d5a173c4427 is 50, key is test_row_0/C:col10/1732195484520/Put/seqid=0 2024-11-21T13:24:46,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742076_1252 (size=12301) 2024-11-21T13:24:46,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/4518dfb99d154be9b82a9d5a173c4427 2024-11-21T13:24:46,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/6c9b92ac225a456f81c04ca93ea895e3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6c9b92ac225a456f81c04ca93ea895e3 2024-11-21T13:24:46,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6c9b92ac225a456f81c04ca93ea895e3, entries=250, sequenceid=478, filesize=47.5 K 2024-11-21T13:24:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/b0247b1ba1e74c36b7a036f3c9f0198a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b0247b1ba1e74c36b7a036f3c9f0198a 2024-11-21T13:24:46,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b0247b1ba1e74c36b7a036f3c9f0198a, entries=150, sequenceid=478, filesize=12.0 K 2024-11-21T13:24:46,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/4518dfb99d154be9b82a9d5a173c4427 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4518dfb99d154be9b82a9d5a173c4427 2024-11-21T13:24:46,081 INFO [Thread-610 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-610. 2024-11-21T13:24:46,081 DEBUG [Thread-610 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:46,081 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:46,081 DEBUG [Thread-610 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:46,081 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:46,082 INFO [Thread-614 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-614. 2024-11-21T13:24:46,082 DEBUG [Thread-614 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:46,082 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:46,082 DEBUG [Thread-614 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:46,082 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:46,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4518dfb99d154be9b82a9d5a173c4427, entries=150, sequenceid=478, filesize=12.0 K 2024-11-21T13:24:46,084 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 932ms, sequenceid=478, compaction requested=true 2024-11-21T13:24:46,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:46,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:24:46,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:46,085 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:46,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:24:46,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:46,085 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:24:46,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0eb6d933356cd6a9ba9fa27a04b9b8ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:24:46,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:46,086 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:46,086 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 112205 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:24:46,086 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/B is initiating minor compaction (all files) 2024-11-21T13:24:46,086 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/A is initiating minor compaction (all files) 2024-11-21T13:24:46,086 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/B in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:46,086 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/A in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:46,086 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/51be5eb3f5d94729811385f19cc30da4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/7930034c1c444b379750f71dab0188ac, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b0247b1ba1e74c36b7a036f3c9f0198a] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=37.1 K 2024-11-21T13:24:46,086 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/b1e1612f8c97493284f2d284639c2fcf, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/8418dc2e242d437783e730307f5bd2a3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6c9b92ac225a456f81c04ca93ea895e3] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=109.6 K 2024-11-21T13:24:46,086 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:46,086 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/b1e1612f8c97493284f2d284639c2fcf, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/8418dc2e242d437783e730307f5bd2a3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6c9b92ac225a456f81c04ca93ea895e3] 2024-11-21T13:24:46,087 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 51be5eb3f5d94729811385f19cc30da4, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732195482970 2024-11-21T13:24:46,087 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1e1612f8c97493284f2d284639c2fcf, keycount=150, bloomtype=ROW, size=31.6 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732195482970 2024-11-21T13:24:46,087 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8418dc2e242d437783e730307f5bd2a3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1732195483333 2024-11-21T13:24:46,087 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 7930034c1c444b379750f71dab0188ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1732195483333 2024-11-21T13:24:46,087 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b0247b1ba1e74c36b7a036f3c9f0198a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732195484520 2024-11-21T13:24:46,087 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c9b92ac225a456f81c04ca93ea895e3, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732195484514 2024-11-21T13:24:46,089 INFO [Thread-616 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-616. 2024-11-21T13:24:46,089 DEBUG [Thread-616 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:46,089 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:46,089 DEBUG [Thread-616 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:46,089 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:46,090 INFO [Thread-612 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-612. 2024-11-21T13:24:46,090 DEBUG [Thread-612 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:46,090 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:46,090 DEBUG [Thread-612 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:46,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:46,097 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#B#compaction#203 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:46,098 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/ae8afa26bbe245fb85070a202bd4dfd1 is 50, key is test_row_0/B:col10/1732195484520/Put/seqid=0 2024-11-21T13:24:46,101 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:46,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-21T13:24:46,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:46,105 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T13:24:46,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742077_1253 (size=13459) 2024-11-21T13:24:46,110 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411214650cdfc36094e7a92ac986dc7236034_0eb6d933356cd6a9ba9fa27a04b9b8ff store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:46,113 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411214650cdfc36094e7a92ac986dc7236034_0eb6d933356cd6a9ba9fa27a04b9b8ff, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:46,113 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214650cdfc36094e7a92ac986dc7236034_0eb6d933356cd6a9ba9fa27a04b9b8ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:46,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112117c9d9edf9884cc3ab113eefbd1dc12a_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195485185/Put/seqid=0 2024-11-21T13:24:46,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742078_1254 (size=4469) 2024-11-21T13:24:46,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742079_1255 (size=12454) 2024-11-21T13:24:46,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:46,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. as already flushing 2024-11-21T13:24:46,334 INFO [Thread-599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-599. 2024-11-21T13:24:46,334 INFO [Thread-601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-601. 2024-11-21T13:24:46,334 DEBUG [Thread-599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:46,334 DEBUG [Thread-601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:46,334 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:46,334 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:46,334 DEBUG [Thread-599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:46,334 DEBUG [Thread-601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:46,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:46,335 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T13:24:46,520 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/ae8afa26bbe245fb85070a202bd4dfd1 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/ae8afa26bbe245fb85070a202bd4dfd1 2024-11-21T13:24:46,526 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#A#compaction#204 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:46,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:46,527 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/0b5d49dfa8164f0e91f81529e4bae17d is 175, key is test_row_0/A:col10/1732195484520/Put/seqid=0 2024-11-21T13:24:46,529 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/B of 0eb6d933356cd6a9ba9fa27a04b9b8ff into ae8afa26bbe245fb85070a202bd4dfd1(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:46,529 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:46,529 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/B, priority=13, startTime=1732195486085; duration=0sec 2024-11-21T13:24:46,529 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:24:46,529 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:B 2024-11-21T13:24:46,529 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:24:46,531 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:24:46,531 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 0eb6d933356cd6a9ba9fa27a04b9b8ff/C is initiating minor compaction (all files) 2024-11-21T13:24:46,531 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0eb6d933356cd6a9ba9fa27a04b9b8ff/C in TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:46,532 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e5a2c24fa5f74262bbc91055e56dbcdb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/fd0cf7ca909d4840a5bedce23a60ced0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/d584a5ae4361448080abb974f7b6e3f0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4518dfb99d154be9b82a9d5a173c4427] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp, totalSize=49.0 K 2024-11-21T13:24:46,532 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e5a2c24fa5f74262bbc91055e56dbcdb, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=419, earliestPutTs=1732195482304 2024-11-21T13:24:46,533 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112117c9d9edf9884cc3ab113eefbd1dc12a_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112117c9d9edf9884cc3ab113eefbd1dc12a_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:46,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742080_1256 (size=32413) 2024-11-21T13:24:46,533 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting fd0cf7ca909d4840a5bedce23a60ced0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1732195482970 2024-11-21T13:24:46,534 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting d584a5ae4361448080abb974f7b6e3f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1732195483333 2024-11-21T13:24:46,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/4391358b49794cd58f7bb167bcff214c, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:46,534 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 4518dfb99d154be9b82a9d5a173c4427, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=478, earliestPutTs=1732195484520 2024-11-21T13:24:46,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/4391358b49794cd58f7bb167bcff214c is 175, key is test_row_0/A:col10/1732195485185/Put/seqid=0 2024-11-21T13:24:46,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742081_1257 (size=31255) 2024-11-21T13:24:46,542 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0eb6d933356cd6a9ba9fa27a04b9b8ff#C#compaction#206 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:24:46,543 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/f693490283f3470793e216f137b6f026 is 50, key is test_row_0/C:col10/1732195484520/Put/seqid=0 2024-11-21T13:24:46,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742082_1258 (size=13425) 2024-11-21T13:24:46,940 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=496, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/4391358b49794cd58f7bb167bcff214c 2024-11-21T13:24:46,944 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/0b5d49dfa8164f0e91f81529e4bae17d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/0b5d49dfa8164f0e91f81529e4bae17d 2024-11-21T13:24:46,950 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/A of 0eb6d933356cd6a9ba9fa27a04b9b8ff into 0b5d49dfa8164f0e91f81529e4bae17d(size=31.7 K), total size for store is 31.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:46,950 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:46,950 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/A, priority=13, startTime=1732195486084; duration=0sec 2024-11-21T13:24:46,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/67cbe1fbd8054fee8e4648ed9547848a is 50, key is test_row_0/B:col10/1732195485185/Put/seqid=0 2024-11-21T13:24:46,950 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:46,950 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:A 2024-11-21T13:24:46,951 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/f693490283f3470793e216f137b6f026 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/f693490283f3470793e216f137b6f026 2024-11-21T13:24:46,956 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0eb6d933356cd6a9ba9fa27a04b9b8ff/C of 0eb6d933356cd6a9ba9fa27a04b9b8ff into f693490283f3470793e216f137b6f026(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:24:46,956 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:46,956 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., storeName=0eb6d933356cd6a9ba9fa27a04b9b8ff/C, priority=12, startTime=1732195486085; duration=0sec 2024-11-21T13:24:46,956 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:24:46,956 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0eb6d933356cd6a9ba9fa27a04b9b8ff:C 2024-11-21T13:24:46,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742083_1259 (size=12301) 2024-11-21T13:24:47,362 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/67cbe1fbd8054fee8e4648ed9547848a 2024-11-21T13:24:47,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/368bbad0f9e847e1943f873006716ab6 is 50, key is test_row_0/C:col10/1732195485185/Put/seqid=0 2024-11-21T13:24:47,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742084_1260 (size=12301) 2024-11-21T13:24:47,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T13:24:47,775 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/368bbad0f9e847e1943f873006716ab6 2024-11-21T13:24:47,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/4391358b49794cd58f7bb167bcff214c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/4391358b49794cd58f7bb167bcff214c 2024-11-21T13:24:47,791 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/4391358b49794cd58f7bb167bcff214c, entries=150, sequenceid=496, filesize=30.5 K 2024-11-21T13:24:47,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/67cbe1fbd8054fee8e4648ed9547848a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/67cbe1fbd8054fee8e4648ed9547848a 2024-11-21T13:24:47,797 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/67cbe1fbd8054fee8e4648ed9547848a, entries=150, sequenceid=496, filesize=12.0 K 2024-11-21T13:24:47,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/368bbad0f9e847e1943f873006716ab6 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/368bbad0f9e847e1943f873006716ab6 2024-11-21T13:24:47,802 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/368bbad0f9e847e1943f873006716ab6, entries=150, sequenceid=496, filesize=12.0 K 2024-11-21T13:24:47,802 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=13.42 KB/13740 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 1697ms, sequenceid=496, compaction requested=false 2024-11-21T13:24:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: 2024-11-21T13:24:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:47,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-21T13:24:47,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-21T13:24:47,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-21T13:24:47,805 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4830 sec 2024-11-21T13:24:47,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.4870 sec 2024-11-21T13:24:49,229 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T13:24:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-21T13:24:49,491 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:55,572 INFO [Thread-607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-607. 2024-11-21T13:24:55,572 DEBUG [Thread-607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:55,572 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:55,572 DEBUG [Thread-607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:55,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:55,631 INFO [Thread-603 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-603. 2024-11-21T13:24:55,631 DEBUG [Thread-603 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:55,631 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:55,631 DEBUG [Thread-603 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:55,631 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:55,641 INFO [Thread-605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-605. 2024-11-21T13:24:55,642 DEBUG [Thread-605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:24:55,642 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:55,642 DEBUG [Thread-605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:55,642 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:55,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T13:24:55,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 180 2024-11-21T13:24:55,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 170 2024-11-21T13:24:55,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-21T13:24:55,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-21T13:24:55,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-21T13:24:55,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T13:24:55,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3516 2024-11-21T13:24:55,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3518 2024-11-21T13:24:55,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T13:24:55,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1568 2024-11-21T13:24:55,642 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4698 rows 2024-11-21T13:24:55,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1561 2024-11-21T13:24:55,643 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4680 rows 2024-11-21T13:24:55,643 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T13:24:55,643 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T13:24:55,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:55,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:24:55,643 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:24:55,645 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T13:24:55,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:55,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T13:24:55,650 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195495650"}]},"ts":"1732195495650"} 2024-11-21T13:24:55,652 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T13:24:55,652 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T13:24:55,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T13:24:55,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, UNASSIGN}] 2024-11-21T13:24:55,656 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, UNASSIGN 2024-11-21T13:24:55,656 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=0eb6d933356cd6a9ba9fa27a04b9b8ff, regionState=CLOSING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:55,657 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, UNASSIGN because future has completed 2024-11-21T13:24:55,658 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T13:24:55,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:24:55,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T13:24:55,811 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:55,811 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T13:24:55,811 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 0eb6d933356cd6a9ba9fa27a04b9b8ff, disabling compactions & flushes 2024-11-21T13:24:55,811 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:55,812 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:55,812 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. after waiting 0 ms 2024-11-21T13:24:55,812 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:55,812 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T13:24:55,824 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a44793756b6049ff89f7a16dcfa1cc02_0eb6d933356cd6a9ba9fa27a04b9b8ff is 50, key is test_row_0/A:col10/1732195495629/Put/seqid=0 2024-11-21T13:24:55,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742085_1261 (size=12454) 2024-11-21T13:24:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T13:24:56,230 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:56,274 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a44793756b6049ff89f7a16dcfa1cc02_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a44793756b6049ff89f7a16dcfa1cc02_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:56,276 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/19a680ffd4ec4831988528ef7089e9c0, store: [table=TestAcidGuarantees family=A region=0eb6d933356cd6a9ba9fa27a04b9b8ff] 2024-11-21T13:24:56,277 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/19a680ffd4ec4831988528ef7089e9c0 is 175, key is test_row_0/A:col10/1732195495629/Put/seqid=0 2024-11-21T13:24:56,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T13:24:56,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742086_1262 (size=31255) 2024-11-21T13:24:56,686 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=507, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/19a680ffd4ec4831988528ef7089e9c0 2024-11-21T13:24:56,700 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/fe5b08ce1fc341919c42e791e9946b77 is 50, key is test_row_0/B:col10/1732195495629/Put/seqid=0 2024-11-21T13:24:56,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742087_1263 (size=12301) 2024-11-21T13:24:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T13:24:57,104 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/fe5b08ce1fc341919c42e791e9946b77 2024-11-21T13:24:57,111 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/b269aa6e1db748c382a4b6522d805b8e is 50, key is test_row_0/C:col10/1732195495629/Put/seqid=0 2024-11-21T13:24:57,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742088_1264 (size=12301) 2024-11-21T13:24:57,515 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/b269aa6e1db748c382a4b6522d805b8e 2024-11-21T13:24:57,521 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/A/19a680ffd4ec4831988528ef7089e9c0 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/19a680ffd4ec4831988528ef7089e9c0 2024-11-21T13:24:57,526 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/19a680ffd4ec4831988528ef7089e9c0, entries=150, sequenceid=507, filesize=30.5 K 2024-11-21T13:24:57,528 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/B/fe5b08ce1fc341919c42e791e9946b77 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/fe5b08ce1fc341919c42e791e9946b77 2024-11-21T13:24:57,534 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/fe5b08ce1fc341919c42e791e9946b77, entries=150, sequenceid=507, filesize=12.0 K 2024-11-21T13:24:57,535 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/.tmp/C/b269aa6e1db748c382a4b6522d805b8e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b269aa6e1db748c382a4b6522d805b8e 2024-11-21T13:24:57,541 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b269aa6e1db748c382a4b6522d805b8e, entries=150, sequenceid=507, filesize=12.0 K 2024-11-21T13:24:57,542 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 1730ms, sequenceid=507, compaction requested=true 2024-11-21T13:24:57,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/d8aa40d2379a4ae9873f5c34b8cb0be1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a976bf05c8084b2aa2a387c53441126b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1c4cd7ee778b43648cdc9973f0244e6d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/7ed22395e0954790ba5c38b9cc359424, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6572e1482f58417fb7bbf620e8561099, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/0cb76b156e2d43978a14bc24d99f216e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e7233eef54474f05b4e4463be5407d77, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/9e303303884d48799b9760521641fe2f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/3664a84017734b2abe9527cd80dd56a6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/14df2e1660d943cfbe6c46dfeeaa5971, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a027c13bbff9490c9d92f4a8b15a2aa8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/fb178672fd544698b720197a2d9a1238, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/4b2ae0e042004c7e9fdf3cf843babb6f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c4a27d6eedb74976b588b45d5d6dc2ef, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/79f33788b9f54c49bcbc3d2688c3ca41, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e2ed7d73167244bda0b3941006555a77, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/580c6d53262c4b70ae69d080d90ab0ed, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c077e4e7732e447c973f6bfd11ecfc46, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c680e970960e451d8d51751482c37ca3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/68b28ce3e63a40849e42abef0e592a5e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/360da96727694637a7fedd7f2fc7322c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1e97d676e3374601a144ad3a68e830b3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/db6de0080b26496aa3de53964a7d8670, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/5b6033ef3eac4e5bacd8e19d03c75a31, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/ab2b5e107e064d0185cd467beabc9882, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1b4e08cb9e6b4be389b975636216fe92, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/da51c61c692b4f1cbb1899da082894a7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/fdcf755c53d1431e8295d48d4ae2f71d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e7c9c321f7e6476e872fae347071f9f9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/dc150fd36a8e47f0b4829d4a2dd90085, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/b1e1612f8c97493284f2d284639c2fcf, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e5cfb3cdab404627881c42e34eac185d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/8418dc2e242d437783e730307f5bd2a3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6c9b92ac225a456f81c04ca93ea895e3] to archive 2024-11-21T13:24:57,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:24:57,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/d8aa40d2379a4ae9873f5c34b8cb0be1 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/d8aa40d2379a4ae9873f5c34b8cb0be1 2024-11-21T13:24:57,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a976bf05c8084b2aa2a387c53441126b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a976bf05c8084b2aa2a387c53441126b 2024-11-21T13:24:57,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1c4cd7ee778b43648cdc9973f0244e6d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1c4cd7ee778b43648cdc9973f0244e6d 2024-11-21T13:24:57,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/7ed22395e0954790ba5c38b9cc359424 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/7ed22395e0954790ba5c38b9cc359424 2024-11-21T13:24:57,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6572e1482f58417fb7bbf620e8561099 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6572e1482f58417fb7bbf620e8561099 2024-11-21T13:24:57,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/0cb76b156e2d43978a14bc24d99f216e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/0cb76b156e2d43978a14bc24d99f216e 2024-11-21T13:24:57,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e7233eef54474f05b4e4463be5407d77 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e7233eef54474f05b4e4463be5407d77 2024-11-21T13:24:57,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/9e303303884d48799b9760521641fe2f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/9e303303884d48799b9760521641fe2f 2024-11-21T13:24:57,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/3664a84017734b2abe9527cd80dd56a6 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/3664a84017734b2abe9527cd80dd56a6 2024-11-21T13:24:57,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/14df2e1660d943cfbe6c46dfeeaa5971 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/14df2e1660d943cfbe6c46dfeeaa5971 2024-11-21T13:24:57,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a027c13bbff9490c9d92f4a8b15a2aa8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/a027c13bbff9490c9d92f4a8b15a2aa8 2024-11-21T13:24:57,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/fb178672fd544698b720197a2d9a1238 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/fb178672fd544698b720197a2d9a1238 2024-11-21T13:24:57,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/4b2ae0e042004c7e9fdf3cf843babb6f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/4b2ae0e042004c7e9fdf3cf843babb6f 2024-11-21T13:24:57,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c4a27d6eedb74976b588b45d5d6dc2ef to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c4a27d6eedb74976b588b45d5d6dc2ef 2024-11-21T13:24:57,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/79f33788b9f54c49bcbc3d2688c3ca41 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/79f33788b9f54c49bcbc3d2688c3ca41 2024-11-21T13:24:57,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e2ed7d73167244bda0b3941006555a77 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e2ed7d73167244bda0b3941006555a77 2024-11-21T13:24:57,577 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/580c6d53262c4b70ae69d080d90ab0ed to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/580c6d53262c4b70ae69d080d90ab0ed 2024-11-21T13:24:57,579 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c077e4e7732e447c973f6bfd11ecfc46 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c077e4e7732e447c973f6bfd11ecfc46 2024-11-21T13:24:57,581 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c680e970960e451d8d51751482c37ca3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/c680e970960e451d8d51751482c37ca3 2024-11-21T13:24:57,582 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/68b28ce3e63a40849e42abef0e592a5e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/68b28ce3e63a40849e42abef0e592a5e 2024-11-21T13:24:57,584 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/360da96727694637a7fedd7f2fc7322c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/360da96727694637a7fedd7f2fc7322c 2024-11-21T13:24:57,585 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1e97d676e3374601a144ad3a68e830b3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1e97d676e3374601a144ad3a68e830b3 2024-11-21T13:24:57,587 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/db6de0080b26496aa3de53964a7d8670 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/db6de0080b26496aa3de53964a7d8670 2024-11-21T13:24:57,592 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/5b6033ef3eac4e5bacd8e19d03c75a31 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/5b6033ef3eac4e5bacd8e19d03c75a31 2024-11-21T13:24:57,594 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/ab2b5e107e064d0185cd467beabc9882 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/ab2b5e107e064d0185cd467beabc9882 2024-11-21T13:24:57,595 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1b4e08cb9e6b4be389b975636216fe92 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/1b4e08cb9e6b4be389b975636216fe92 2024-11-21T13:24:57,597 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/da51c61c692b4f1cbb1899da082894a7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/da51c61c692b4f1cbb1899da082894a7 2024-11-21T13:24:57,598 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/fdcf755c53d1431e8295d48d4ae2f71d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/fdcf755c53d1431e8295d48d4ae2f71d 2024-11-21T13:24:57,600 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e7c9c321f7e6476e872fae347071f9f9 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e7c9c321f7e6476e872fae347071f9f9 2024-11-21T13:24:57,601 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/dc150fd36a8e47f0b4829d4a2dd90085 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/dc150fd36a8e47f0b4829d4a2dd90085 2024-11-21T13:24:57,603 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/b1e1612f8c97493284f2d284639c2fcf to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/b1e1612f8c97493284f2d284639c2fcf 2024-11-21T13:24:57,604 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e5cfb3cdab404627881c42e34eac185d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/e5cfb3cdab404627881c42e34eac185d 2024-11-21T13:24:57,605 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/8418dc2e242d437783e730307f5bd2a3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/8418dc2e242d437783e730307f5bd2a3 2024-11-21T13:24:57,607 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6c9b92ac225a456f81c04ca93ea895e3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/6c9b92ac225a456f81c04ca93ea895e3 2024-11-21T13:24:57,609 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c42da8269ef44cdfb9d61c62ef0f5add, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/65a316cca9dc4e31b1d75a7a679e815b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/114df3f6f44e4ea3811ae1f28a09b00a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/f567dd2203ad4149a666f3fd33bf1ddc, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b48a8f6dc1c04fb1b7a511ea3e5f4322, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/ef2834da0d904edd90c4bd8a2ada69e5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/145b3618371f4b63906e671fa3000329, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/a3ada867e453428dbdc10c6123255025, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/7318a7b7845443baabbe8b456146513c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/9206e0af54604ebd89a504635dadf1ad, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/9d1240731985490faab39074f5fb54f3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/5a2d551e560f4991ba9b70f6021b2aea, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c01f76e27c7b4bd382e43f018d9b1d79, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/1224d006e8f14a1588b5a80d87aefcbc, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c24d845309d14746b282e650b7cdec8c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/1bcd4ca9880844269182e7095ee1a480, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/598edc29499e40278afa9ee6c788a239, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b2fa84847ec746798dc1c3dfc1ce7a0b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/4dcf0d062c6148db9a226c12e43a17ff, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/f4f1a25c37f843899ed0d585a0d4a62c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/e3a4793032de46c78a1c307e07b2c705, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/63ba5c5795d1487b88d45dd546b1920d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b287491a1f6a401cbc96d1e968a09661, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b46b2bf1a8984b4dbae0eff91dddab24, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b1825bd09a764513baab0f269dfff8ea, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/86d2980284fa44ccad19c9dab18b040f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/64e53110ce3e497bafaf49b66a725a12, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/bd5f5555e093429f90dd8b90eaa66a53, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/26024d6270f34d6da3e49797cdafb475, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/176fe43c31de480c8654ab97d022349a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/51be5eb3f5d94729811385f19cc30da4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/49f792811e354347bbd63302b88e840f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/7930034c1c444b379750f71dab0188ac, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b0247b1ba1e74c36b7a036f3c9f0198a] to archive 2024-11-21T13:24:57,610 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:24:57,612 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c42da8269ef44cdfb9d61c62ef0f5add to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c42da8269ef44cdfb9d61c62ef0f5add 2024-11-21T13:24:57,613 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/65a316cca9dc4e31b1d75a7a679e815b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/65a316cca9dc4e31b1d75a7a679e815b 2024-11-21T13:24:57,614 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/114df3f6f44e4ea3811ae1f28a09b00a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/114df3f6f44e4ea3811ae1f28a09b00a 2024-11-21T13:24:57,616 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/f567dd2203ad4149a666f3fd33bf1ddc to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/f567dd2203ad4149a666f3fd33bf1ddc 2024-11-21T13:24:57,617 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b48a8f6dc1c04fb1b7a511ea3e5f4322 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b48a8f6dc1c04fb1b7a511ea3e5f4322 2024-11-21T13:24:57,621 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/ef2834da0d904edd90c4bd8a2ada69e5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/ef2834da0d904edd90c4bd8a2ada69e5 2024-11-21T13:24:57,622 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/145b3618371f4b63906e671fa3000329 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/145b3618371f4b63906e671fa3000329 2024-11-21T13:24:57,624 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/a3ada867e453428dbdc10c6123255025 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/a3ada867e453428dbdc10c6123255025 2024-11-21T13:24:57,625 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/7318a7b7845443baabbe8b456146513c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/7318a7b7845443baabbe8b456146513c 2024-11-21T13:24:57,626 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/9206e0af54604ebd89a504635dadf1ad to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/9206e0af54604ebd89a504635dadf1ad 2024-11-21T13:24:57,627 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/9d1240731985490faab39074f5fb54f3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/9d1240731985490faab39074f5fb54f3 2024-11-21T13:24:57,628 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/5a2d551e560f4991ba9b70f6021b2aea to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/5a2d551e560f4991ba9b70f6021b2aea 2024-11-21T13:24:57,629 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c01f76e27c7b4bd382e43f018d9b1d79 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c01f76e27c7b4bd382e43f018d9b1d79 2024-11-21T13:24:57,630 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/1224d006e8f14a1588b5a80d87aefcbc to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/1224d006e8f14a1588b5a80d87aefcbc 2024-11-21T13:24:57,631 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c24d845309d14746b282e650b7cdec8c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/c24d845309d14746b282e650b7cdec8c 2024-11-21T13:24:57,632 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/1bcd4ca9880844269182e7095ee1a480 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/1bcd4ca9880844269182e7095ee1a480 2024-11-21T13:24:57,633 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/598edc29499e40278afa9ee6c788a239 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/598edc29499e40278afa9ee6c788a239 2024-11-21T13:24:57,634 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b2fa84847ec746798dc1c3dfc1ce7a0b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b2fa84847ec746798dc1c3dfc1ce7a0b 2024-11-21T13:24:57,635 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/4dcf0d062c6148db9a226c12e43a17ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/4dcf0d062c6148db9a226c12e43a17ff 2024-11-21T13:24:57,636 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/f4f1a25c37f843899ed0d585a0d4a62c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/f4f1a25c37f843899ed0d585a0d4a62c 2024-11-21T13:24:57,637 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/e3a4793032de46c78a1c307e07b2c705 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/e3a4793032de46c78a1c307e07b2c705 2024-11-21T13:24:57,638 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/63ba5c5795d1487b88d45dd546b1920d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/63ba5c5795d1487b88d45dd546b1920d 2024-11-21T13:24:57,639 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b287491a1f6a401cbc96d1e968a09661 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b287491a1f6a401cbc96d1e968a09661 2024-11-21T13:24:57,641 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b46b2bf1a8984b4dbae0eff91dddab24 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b46b2bf1a8984b4dbae0eff91dddab24 2024-11-21T13:24:57,644 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b1825bd09a764513baab0f269dfff8ea to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b1825bd09a764513baab0f269dfff8ea 2024-11-21T13:24:57,646 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/86d2980284fa44ccad19c9dab18b040f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/86d2980284fa44ccad19c9dab18b040f 2024-11-21T13:24:57,647 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/64e53110ce3e497bafaf49b66a725a12 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/64e53110ce3e497bafaf49b66a725a12 2024-11-21T13:24:57,648 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/bd5f5555e093429f90dd8b90eaa66a53 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/bd5f5555e093429f90dd8b90eaa66a53 2024-11-21T13:24:57,649 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/26024d6270f34d6da3e49797cdafb475 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/26024d6270f34d6da3e49797cdafb475 2024-11-21T13:24:57,650 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/176fe43c31de480c8654ab97d022349a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/176fe43c31de480c8654ab97d022349a 2024-11-21T13:24:57,652 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/51be5eb3f5d94729811385f19cc30da4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/51be5eb3f5d94729811385f19cc30da4 2024-11-21T13:24:57,653 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/49f792811e354347bbd63302b88e840f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/49f792811e354347bbd63302b88e840f 2024-11-21T13:24:57,654 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/7930034c1c444b379750f71dab0188ac to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/7930034c1c444b379750f71dab0188ac 2024-11-21T13:24:57,655 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b0247b1ba1e74c36b7a036f3c9f0198a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/b0247b1ba1e74c36b7a036f3c9f0198a 2024-11-21T13:24:57,657 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/12c4b107dbdc4ccebb44bf9133804bae, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/753a8eeb85ed4bd587a224198d015595, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/39ce641f90774451849db558d3c6731e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/051b1560c69c4a0e9ae94fca3ee208d8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/c0533ac10f8d4193be616b38473999aa, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/cf1403eecdb546269b9e7b108d1b4d95, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/500cc87e447c482783a73759a79deb3b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/3b50eb35490e42b4876ec4447e961a57, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/df5214e2de1e47989010ac9bf7b16404, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/068cad575c8b417ea21b5c083a9bcbee, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/145bfb3e4aee4a91839ceebe0e98c384, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b476e9d113414282a721979259606f3e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/026e72f93b6f42a99a3e0c3ed7600220, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/3b2e81ec6f8046969660a9950d7f69e4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/f260ca17a652430cbf03937a2f5111b1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b97b376535d6445ca6280fdd9ba7af00, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4de0be26117342c0a861f86ac9fc94a8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e49f352c6e624248bb9f826ffbf12df4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/de1d11d939a24cd88a81d9c849135af9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/38005dc928624fffa63f77d67c2cd28f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/171b6ce62e98477fb5a075167ccd7447, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/fe8c798bde6b4822b402697eb05c1438, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4ca166dd258b4627b9c5c32a064bed19, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/ca3f204cd2fc4bc091cceac06ce47d58, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/9ecebb3a697c4f41b8f6d298c856f4c8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e94014bb7aa44ebc973f10bcbb6455ab, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/24440117f333478cb1548d3565914011, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/31a1fc8a959546fa80ef87c72429e0e4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e5a2c24fa5f74262bbc91055e56dbcdb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/957af8775242408f9a4c44c45ba1df1b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/fd0cf7ca909d4840a5bedce23a60ced0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/d584a5ae4361448080abb974f7b6e3f0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4518dfb99d154be9b82a9d5a173c4427] to archive 2024-11-21T13:24:57,659 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:24:57,660 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/12c4b107dbdc4ccebb44bf9133804bae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/12c4b107dbdc4ccebb44bf9133804bae 2024-11-21T13:24:57,662 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/753a8eeb85ed4bd587a224198d015595 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/753a8eeb85ed4bd587a224198d015595 2024-11-21T13:24:57,663 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/39ce641f90774451849db558d3c6731e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/39ce641f90774451849db558d3c6731e 2024-11-21T13:24:57,664 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/051b1560c69c4a0e9ae94fca3ee208d8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/051b1560c69c4a0e9ae94fca3ee208d8 2024-11-21T13:24:57,665 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/c0533ac10f8d4193be616b38473999aa to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/c0533ac10f8d4193be616b38473999aa 2024-11-21T13:24:57,666 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/cf1403eecdb546269b9e7b108d1b4d95 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/cf1403eecdb546269b9e7b108d1b4d95 2024-11-21T13:24:57,668 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/500cc87e447c482783a73759a79deb3b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/500cc87e447c482783a73759a79deb3b 2024-11-21T13:24:57,670 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/3b50eb35490e42b4876ec4447e961a57 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/3b50eb35490e42b4876ec4447e961a57 2024-11-21T13:24:57,671 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/df5214e2de1e47989010ac9bf7b16404 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/df5214e2de1e47989010ac9bf7b16404 2024-11-21T13:24:57,672 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/068cad575c8b417ea21b5c083a9bcbee to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/068cad575c8b417ea21b5c083a9bcbee 2024-11-21T13:24:57,674 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/145bfb3e4aee4a91839ceebe0e98c384 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/145bfb3e4aee4a91839ceebe0e98c384 2024-11-21T13:24:57,675 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b476e9d113414282a721979259606f3e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b476e9d113414282a721979259606f3e 2024-11-21T13:24:57,676 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/026e72f93b6f42a99a3e0c3ed7600220 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/026e72f93b6f42a99a3e0c3ed7600220 2024-11-21T13:24:57,678 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/3b2e81ec6f8046969660a9950d7f69e4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/3b2e81ec6f8046969660a9950d7f69e4 2024-11-21T13:24:57,679 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/f260ca17a652430cbf03937a2f5111b1 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/f260ca17a652430cbf03937a2f5111b1 2024-11-21T13:24:57,680 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b97b376535d6445ca6280fdd9ba7af00 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b97b376535d6445ca6280fdd9ba7af00 2024-11-21T13:24:57,682 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4de0be26117342c0a861f86ac9fc94a8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4de0be26117342c0a861f86ac9fc94a8 2024-11-21T13:24:57,683 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e49f352c6e624248bb9f826ffbf12df4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e49f352c6e624248bb9f826ffbf12df4 2024-11-21T13:24:57,684 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/de1d11d939a24cd88a81d9c849135af9 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/de1d11d939a24cd88a81d9c849135af9 2024-11-21T13:24:57,686 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/38005dc928624fffa63f77d67c2cd28f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/38005dc928624fffa63f77d67c2cd28f 2024-11-21T13:24:57,688 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/171b6ce62e98477fb5a075167ccd7447 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/171b6ce62e98477fb5a075167ccd7447 2024-11-21T13:24:57,689 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/fe8c798bde6b4822b402697eb05c1438 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/fe8c798bde6b4822b402697eb05c1438 2024-11-21T13:24:57,690 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4ca166dd258b4627b9c5c32a064bed19 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4ca166dd258b4627b9c5c32a064bed19 2024-11-21T13:24:57,692 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/ca3f204cd2fc4bc091cceac06ce47d58 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/ca3f204cd2fc4bc091cceac06ce47d58 2024-11-21T13:24:57,693 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/9ecebb3a697c4f41b8f6d298c856f4c8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/9ecebb3a697c4f41b8f6d298c856f4c8 2024-11-21T13:24:57,694 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e94014bb7aa44ebc973f10bcbb6455ab to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e94014bb7aa44ebc973f10bcbb6455ab 2024-11-21T13:24:57,695 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/24440117f333478cb1548d3565914011 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/24440117f333478cb1548d3565914011 2024-11-21T13:24:57,696 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/31a1fc8a959546fa80ef87c72429e0e4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/31a1fc8a959546fa80ef87c72429e0e4 2024-11-21T13:24:57,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e5a2c24fa5f74262bbc91055e56dbcdb to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/e5a2c24fa5f74262bbc91055e56dbcdb 2024-11-21T13:24:57,697 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/957af8775242408f9a4c44c45ba1df1b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/957af8775242408f9a4c44c45ba1df1b 2024-11-21T13:24:57,698 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/fd0cf7ca909d4840a5bedce23a60ced0 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/fd0cf7ca909d4840a5bedce23a60ced0 2024-11-21T13:24:57,699 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/d584a5ae4361448080abb974f7b6e3f0 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/d584a5ae4361448080abb974f7b6e3f0 2024-11-21T13:24:57,700 DEBUG [StoreCloser-TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4518dfb99d154be9b82a9d5a173c4427 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/4518dfb99d154be9b82a9d5a173c4427 2024-11-21T13:24:57,705 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/recovered.edits/510.seqid, newMaxSeqId=510, maxSeqId=4 2024-11-21T13:24:57,706 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. 2024-11-21T13:24:57,706 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 0eb6d933356cd6a9ba9fa27a04b9b8ff: Waiting for close lock at 1732195495811Running coprocessor pre-close hooks at 1732195495811Disabling compacts and flushes for region at 1732195495811Disabling writes for close at 1732195495812 (+1 ms)Obtaining lock to block concurrent updates at 1732195495812Preparing flush snapshotting stores in 0eb6d933356cd6a9ba9fa27a04b9b8ff at 1732195495812Finished memstore snapshotting TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732195495813 (+1 ms)Flushing stores of TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff. at 1732195495814 (+1 ms)Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff/A: creating writer at 1732195495814Flushing mob file 0eb6d933356cd6a9ba9fa27a04b9b8ff/A: appending metadata at 1732195495823 (+9 ms)Flushing mob file 0eb6d933356cd6a9ba9fa27a04b9b8ff/A: closing flushed file at 1732195495823Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff/A: appending metadata at 1732195496276 (+453 ms)Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff/A: closing flushed file at 1732195496276Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff/B: creating writer at 1732195496696 (+420 ms)Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff/B: appending metadata at 1732195496699 (+3 ms)Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff/B: closing flushed file at 1732195496699Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff/C: creating writer at 1732195497108 (+409 ms)Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff/C: appending metadata at 1732195497110 (+2 ms)Flushing 0eb6d933356cd6a9ba9fa27a04b9b8ff/C: closing flushed file at 1732195497110Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@125024ac: reopening flushed file at 1732195497520 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b197e0f: reopening flushed file at 1732195497527 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a783835: reopening flushed file at 1732195497534 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0eb6d933356cd6a9ba9fa27a04b9b8ff in 1730ms, sequenceid=507, compaction requested=true at 1732195497542 (+8 ms)Writing region close event to WAL at 1732195497702 (+160 ms)Running coprocessor post-close hooks at 1732195497706 (+4 ms)Closed at 1732195497706 2024-11-21T13:24:57,708 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,709 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=0eb6d933356cd6a9ba9fa27a04b9b8ff, regionState=CLOSED 2024-11-21T13:24:57,711 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:24:57,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-21T13:24:57,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 0eb6d933356cd6a9ba9fa27a04b9b8ff, server=4fe6efa1e611,33573,1732195430851 in 2.0540 sec 2024-11-21T13:24:57,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-21T13:24:57,717 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0eb6d933356cd6a9ba9fa27a04b9b8ff, UNASSIGN in 2.0600 sec 2024-11-21T13:24:57,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-21T13:24:57,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0650 sec 2024-11-21T13:24:57,721 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195497721"}]},"ts":"1732195497721"} 2024-11-21T13:24:57,723 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T13:24:57,723 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T13:24:57,725 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0790 sec 2024-11-21T13:24:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-21T13:24:57,809 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:57,810 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T13:24:57,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:57,812 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T13:24:57,813 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:57,816 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,819 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/recovered.edits] 2024-11-21T13:24:57,822 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/0b5d49dfa8164f0e91f81529e4bae17d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/0b5d49dfa8164f0e91f81529e4bae17d 2024-11-21T13:24:57,824 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/19a680ffd4ec4831988528ef7089e9c0 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/19a680ffd4ec4831988528ef7089e9c0 2024-11-21T13:24:57,825 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/4391358b49794cd58f7bb167bcff214c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/A/4391358b49794cd58f7bb167bcff214c 2024-11-21T13:24:57,828 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/67cbe1fbd8054fee8e4648ed9547848a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/67cbe1fbd8054fee8e4648ed9547848a 2024-11-21T13:24:57,830 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/ae8afa26bbe245fb85070a202bd4dfd1 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/ae8afa26bbe245fb85070a202bd4dfd1 2024-11-21T13:24:57,831 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/fe5b08ce1fc341919c42e791e9946b77 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/B/fe5b08ce1fc341919c42e791e9946b77 2024-11-21T13:24:57,834 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/368bbad0f9e847e1943f873006716ab6 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/368bbad0f9e847e1943f873006716ab6 2024-11-21T13:24:57,835 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b269aa6e1db748c382a4b6522d805b8e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/b269aa6e1db748c382a4b6522d805b8e 2024-11-21T13:24:57,837 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/f693490283f3470793e216f137b6f026 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/C/f693490283f3470793e216f137b6f026 2024-11-21T13:24:57,840 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/recovered.edits/510.seqid to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff/recovered.edits/510.seqid 2024-11-21T13:24:57,840 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,840 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T13:24:57,841 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T13:24:57,842 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T13:24:57,847 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112108564d35e98541d1b2c83e2cff69a221_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112108564d35e98541d1b2c83e2cff69a221_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,848 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210dfc9c2ad51c405483dce0077c40d01a_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210dfc9c2ad51c405483dce0077c40d01a_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,850 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112117c9d9edf9884cc3ab113eefbd1dc12a_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112117c9d9edf9884cc3ab113eefbd1dc12a_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,851 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212ead4d37d3ac4fb1b4b3ddcd86766c9a_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212ead4d37d3ac4fb1b4b3ddcd86766c9a_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,855 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121327b6f190f2e4f4aa67dc43e4ba156f7_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121327b6f190f2e4f4aa67dc43e4ba156f7_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,857 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112141ae4793ee1a4d779d23cf66bdc71ace_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112141ae4793ee1a4d779d23cf66bdc71ace_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,859 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214f16c072576e43678238600945cc11f8_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214f16c072576e43678238600945cc11f8_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,861 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112150840bd6d383489b8a7a98afcd70ac66_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112150840bd6d383489b8a7a98afcd70ac66_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,862 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112165aac69c4aa74e1791da87fcc21a1939_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112165aac69c4aa74e1791da87fcc21a1939_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,864 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216a3cd89c633742f88ecc1e867921785c_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216a3cd89c633742f88ecc1e867921785c_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,866 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121794c0a7f06b947838a7fafb3846a43fd_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121794c0a7f06b947838a7fafb3846a43fd_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,867 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217e0b1a64db214c08a472b5161354366d_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217e0b1a64db214c08a472b5161354366d_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,869 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218106792790434646a02673d9d0436531_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411218106792790434646a02673d9d0436531_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,872 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121858e4a2829634cbfa96a1353bfa1ccea_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121858e4a2829634cbfa96a1353bfa1ccea_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,874 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121914105d0802e4008bc64a4ee7ade6d10_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121914105d0802e4008bc64a4ee7ade6d10_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,875 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112195184411d4d44b2782311a74d828a11d_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112195184411d4d44b2782311a74d828a11d_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,877 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121970fac9cf5d04409a47238e6c74fc07e_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121970fac9cf5d04409a47238e6c74fc07e_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,878 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a44793756b6049ff89f7a16dcfa1cc02_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a44793756b6049ff89f7a16dcfa1cc02_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,880 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121abbb017edeeb4fe78695d4184bbd894c_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121abbb017edeeb4fe78695d4184bbd894c_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,881 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b3105ca5c1234973b6afaaa5c89138de_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b3105ca5c1234973b6afaaa5c89138de_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,883 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b69469969e1142bc86f82d3aed89783f_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b69469969e1142bc86f82d3aed89783f_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,884 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c4f7c6bc23a7497aadd6fc3f076aa867_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c4f7c6bc23a7497aadd6fc3f076aa867_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,885 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c4f88c337db24f65a4aa4a23f99bf7cd_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c4f88c337db24f65a4aa4a23f99bf7cd_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,886 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cdd0ab1ecc0240b6b7629e25a6fa7f2b_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cdd0ab1ecc0240b6b7629e25a6fa7f2b_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,888 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d0ddb9b8d79946bdb97791393e804702_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d0ddb9b8d79946bdb97791393e804702_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,889 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121de51a4df293a45d981f1d3c4f1fec7e5_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121de51a4df293a45d981f1d3c4f1fec7e5_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,890 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fc204dc8e9734f39ad7288afe42a6748_0eb6d933356cd6a9ba9fa27a04b9b8ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fc204dc8e9734f39ad7288afe42a6748_0eb6d933356cd6a9ba9fa27a04b9b8ff 2024-11-21T13:24:57,891 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T13:24:57,894 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:57,897 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T13:24:57,900 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T13:24:57,902 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:57,902 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T13:24:57,902 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732195497902"}]},"ts":"9223372036854775807"} 2024-11-21T13:24:57,906 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T13:24:57,906 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0eb6d933356cd6a9ba9fa27a04b9b8ff, NAME => 'TestAcidGuarantees,,1732195462818.0eb6d933356cd6a9ba9fa27a04b9b8ff.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T13:24:57,906 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T13:24:57,906 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732195497906"}]},"ts":"9223372036854775807"} 2024-11-21T13:24:57,909 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T13:24:57,910 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:57,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 100 msec 2024-11-21T13:24:57,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-21T13:24:57,920 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T13:24:57,920 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:24:57,932 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 239), OpenFileDescriptor=449 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=495 (was 607), ProcessCount=11 (was 11), AvailableMemoryMB=3166 (was 2778) - AvailableMemoryMB LEAK? - 2024-11-21T13:24:57,943 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=3165 2024-11-21T13:24:57,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T13:24:57,945 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T13:24:57,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T13:24:57,947 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T13:24:57,947 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:57,947 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-21T13:24:57,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T13:24:57,948 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T13:24:57,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742089_1265 (size=959) 2024-11-21T13:24:58,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T13:24:58,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T13:24:58,364 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48 2024-11-21T13:24:58,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742090_1266 (size=53) 2024-11-21T13:24:58,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T13:24:58,771 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:24:58,771 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 3fb1b1f418508c88685394456e36ab0b, disabling compactions & flushes 2024-11-21T13:24:58,771 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:24:58,771 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:24:58,771 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. after waiting 0 ms 2024-11-21T13:24:58,771 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:24:58,771 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:24:58,771 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3fb1b1f418508c88685394456e36ab0b: Waiting for close lock at 1732195498771Disabling compacts and flushes for region at 1732195498771Disabling writes for close at 1732195498771Writing region close event to WAL at 1732195498771Closed at 1732195498771 2024-11-21T13:24:58,773 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T13:24:58,773 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732195498773"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732195498773"}]},"ts":"1732195498773"} 2024-11-21T13:24:58,775 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T13:24:58,776 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T13:24:58,776 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195498776"}]},"ts":"1732195498776"} 2024-11-21T13:24:58,778 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T13:24:58,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3fb1b1f418508c88685394456e36ab0b, ASSIGN}] 2024-11-21T13:24:58,780 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3fb1b1f418508c88685394456e36ab0b, ASSIGN 2024-11-21T13:24:58,780 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3fb1b1f418508c88685394456e36ab0b, ASSIGN; state=OFFLINE, location=4fe6efa1e611,33573,1732195430851; forceNewPlan=false, retain=false 2024-11-21T13:24:58,931 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=3fb1b1f418508c88685394456e36ab0b, regionState=OPENING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:58,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3fb1b1f418508c88685394456e36ab0b, ASSIGN because future has completed 2024-11-21T13:24:58,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:24:59,088 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:24:59,088 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} 2024-11-21T13:24:59,089 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:24:59,089 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:24:59,089 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:24:59,089 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:24:59,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T13:24:59,090 INFO [StoreOpener-3fb1b1f418508c88685394456e36ab0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:24:59,091 INFO [StoreOpener-3fb1b1f418508c88685394456e36ab0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fb1b1f418508c88685394456e36ab0b columnFamilyName A 2024-11-21T13:24:59,091 DEBUG [StoreOpener-3fb1b1f418508c88685394456e36ab0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:59,092 INFO [StoreOpener-3fb1b1f418508c88685394456e36ab0b-1 {}] regionserver.HStore(327): Store=3fb1b1f418508c88685394456e36ab0b/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:24:59,092 INFO [StoreOpener-3fb1b1f418508c88685394456e36ab0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:24:59,093 INFO [StoreOpener-3fb1b1f418508c88685394456e36ab0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fb1b1f418508c88685394456e36ab0b columnFamilyName B 2024-11-21T13:24:59,093 DEBUG [StoreOpener-3fb1b1f418508c88685394456e36ab0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:59,093 INFO [StoreOpener-3fb1b1f418508c88685394456e36ab0b-1 {}] regionserver.HStore(327): Store=3fb1b1f418508c88685394456e36ab0b/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:24:59,093 INFO [StoreOpener-3fb1b1f418508c88685394456e36ab0b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:24:59,094 INFO [StoreOpener-3fb1b1f418508c88685394456e36ab0b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3fb1b1f418508c88685394456e36ab0b columnFamilyName C 2024-11-21T13:24:59,094 DEBUG [StoreOpener-3fb1b1f418508c88685394456e36ab0b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:24:59,094 INFO [StoreOpener-3fb1b1f418508c88685394456e36ab0b-1 {}] regionserver.HStore(327): Store=3fb1b1f418508c88685394456e36ab0b/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:24:59,095 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:24:59,095 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:24:59,096 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:24:59,097 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:24:59,097 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:24:59,097 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T13:24:59,098 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:24:59,100 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T13:24:59,100 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 3fb1b1f418508c88685394456e36ab0b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68125886, jitterRate=0.015154808759689331}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T13:24:59,100 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:24:59,101 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 3fb1b1f418508c88685394456e36ab0b: Running coprocessor pre-open hook at 1732195499089Writing region info on filesystem at 1732195499089Initializing all the Stores at 1732195499090 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195499090Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195499090Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195499090Cleaning up temporary data from old regions at 1732195499097 (+7 ms)Running coprocessor post-open hooks at 1732195499100 (+3 ms)Region opened successfully at 1732195499101 (+1 ms) 2024-11-21T13:24:59,102 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., pid=63, masterSystemTime=1732195499086 2024-11-21T13:24:59,103 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:24:59,103 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:24:59,104 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=3fb1b1f418508c88685394456e36ab0b, regionState=OPEN, openSeqNum=2, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:24:59,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:24:59,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-21T13:24:59,107 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 in 172 msec 2024-11-21T13:24:59,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-21T13:24:59,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3fb1b1f418508c88685394456e36ab0b, ASSIGN in 329 msec 2024-11-21T13:24:59,109 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T13:24:59,109 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195499109"}]},"ts":"1732195499109"} 2024-11-21T13:24:59,111 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T13:24:59,111 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T13:24:59,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1660 sec 2024-11-21T13:25:00,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-21T13:25:00,100 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:00,101 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@578a75e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,101 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:00,102 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:00,103 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:00,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bdf6af4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:00,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:00,105 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57912, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:00,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ca66ed8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,107 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:00,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:00,109 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:00,110 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T13:25:00,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 4fe6efa1e611,46031,1732195430123 2024-11-21T13:25:00,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2191a235 2024-11-21T13:25:00,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T13:25:00,112 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T13:25:00,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b76211b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,114 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:00,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:00,115 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:00,115 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c508760, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:00,116 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:00,117 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:00,117 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57952, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:00,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8920d3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,119 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b1e8a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,119 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:00,120 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:00,121 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b715234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:00,122 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57968, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a00ee33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@509857a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:00,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:00,126 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f7a3c14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:00,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:00,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:00,128 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:00,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c6ca4a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1263b212, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,129 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:00,129 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:00,130 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c3b80fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:00,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:00,132 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:00,132 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57998, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:00,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c7232a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43295e22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:00,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:00,135 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:00,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c1dd201, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:00,136 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:00,137 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:00,138 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7037e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c19450f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,140 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:00,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:00,142 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43636852, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,142 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:00,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:00,144 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:00,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52a41648, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21598d4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,146 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:00,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:00,147 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4e0260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:00,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:00,149 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:00,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30ac3376, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,150 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7bbf3b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,150 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:00,150 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:00,151 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:00,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1929c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:00,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:00,157 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:00,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d7700c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ab4eba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,161 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:00,161 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:00,162 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@673a1449, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:00,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:00,165 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58092, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:00,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e7b592e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11b71713, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:00,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:00,170 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31d502f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,170 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:00,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:00,171 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:00,172 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58112, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:00,173 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39ef7efa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:00,173 DEBUG [Thread-1229 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:00,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:00,175 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:00,175 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46720, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:00,176 DEBUG [Thread-1242 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:00,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:00,178 DEBUG [Thread-1223 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:00,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:00,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:00,180 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:00,201 DEBUG [Thread-1238 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:00,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:00,204 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:00,206 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:00,216 DEBUG [Thread-1234 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:00,217 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:00,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:00,219 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:00,219 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:00,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:25:00,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:00,224 DEBUG [Thread-1225 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:00,225 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-21T13:25:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:00,226 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T13:25:00,227 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:00,228 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:00,228 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:00,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:00,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:00,233 DEBUG [Thread-1240 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:00,235 DEBUG [Thread-1227 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:00,236 DEBUG [Thread-1231 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:00,236 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:00,237 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:00,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46774, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:00,238 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:00,239 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:00,240 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:00,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:00,241 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:00,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:00,242 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:00,244 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:00,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/ce0db2de02834cb5ae72b1010d94a484 is 50, key is test_row_0/A:col10/1732195500182/Put/seqid=0 2024-11-21T13:25:00,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195560250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195560251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,254 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,254 DEBUG [Thread-1236 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:00,254 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195560252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,255 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:00,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,256 DEBUG [Thread-1236 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195560254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195560254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46806, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:00,260 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:00,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742091_1267 (size=16681) 2024-11-21T13:25:00,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/ce0db2de02834cb5ae72b1010d94a484 2024-11-21T13:25:00,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/74a66cab894b47798422ba716040aad8 is 50, key is test_row_0/B:col10/1732195500182/Put/seqid=0 2024-11-21T13:25:00,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T13:25:00,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742092_1268 (size=12001) 2024-11-21T13:25:00,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/74a66cab894b47798422ba716040aad8 2024-11-21T13:25:00,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195560359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195560360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,363 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195560360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195560360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195560360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T13:25:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:00,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:00,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:00,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:00,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/819411b05efd401182dab0dcfd69b54a is 50, key is test_row_0/C:col10/1732195500182/Put/seqid=0 2024-11-21T13:25:00,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742093_1269 (size=12001) 2024-11-21T13:25:00,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/819411b05efd401182dab0dcfd69b54a 2024-11-21T13:25:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/ce0db2de02834cb5ae72b1010d94a484 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/ce0db2de02834cb5ae72b1010d94a484 2024-11-21T13:25:00,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/ce0db2de02834cb5ae72b1010d94a484, entries=250, sequenceid=12, filesize=16.3 K 2024-11-21T13:25:00,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/74a66cab894b47798422ba716040aad8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/74a66cab894b47798422ba716040aad8 2024-11-21T13:25:00,432 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/74a66cab894b47798422ba716040aad8, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T13:25:00,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/819411b05efd401182dab0dcfd69b54a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/819411b05efd401182dab0dcfd69b54a 2024-11-21T13:25:00,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/819411b05efd401182dab0dcfd69b54a, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T13:25:00,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3fb1b1f418508c88685394456e36ab0b in 220ms, sequenceid=12, compaction requested=false 2024-11-21T13:25:00,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T13:25:00,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-21T13:25:00,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:00,542 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T13:25:00,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/caf1e6475d0c432086f041b9db2dad21 is 50, key is test_row_0/A:col10/1732195500248/Put/seqid=0 2024-11-21T13:25:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:00,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:00,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742094_1270 (size=12001) 2024-11-21T13:25:00,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195560574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195560575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195560576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195560577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195560578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195560689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195560689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195560689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195560690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195560690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,693 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T13:25:00,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195560899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195560899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,902 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195560899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195560899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195560900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:00,905 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:00,974 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/caf1e6475d0c432086f041b9db2dad21 2024-11-21T13:25:00,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/9da6a8f6337b4c7dbee0675037fe0e61 is 50, key is test_row_0/B:col10/1732195500248/Put/seqid=0 2024-11-21T13:25:01,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742095_1271 (size=12001) 2024-11-21T13:25:01,016 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/9da6a8f6337b4c7dbee0675037fe0e61 2024-11-21T13:25:01,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/10e6a4a13f494613b607f4f75b2e5c67 is 50, key is test_row_0/C:col10/1732195500248/Put/seqid=0 2024-11-21T13:25:01,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742096_1272 (size=12001) 2024-11-21T13:25:01,029 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/10e6a4a13f494613b607f4f75b2e5c67 2024-11-21T13:25:01,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/caf1e6475d0c432086f041b9db2dad21 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/caf1e6475d0c432086f041b9db2dad21 2024-11-21T13:25:01,042 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/caf1e6475d0c432086f041b9db2dad21, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T13:25:01,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/9da6a8f6337b4c7dbee0675037fe0e61 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9da6a8f6337b4c7dbee0675037fe0e61 2024-11-21T13:25:01,065 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9da6a8f6337b4c7dbee0675037fe0e61, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T13:25:01,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/10e6a4a13f494613b607f4f75b2e5c67 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/10e6a4a13f494613b607f4f75b2e5c67 2024-11-21T13:25:01,073 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/10e6a4a13f494613b607f4f75b2e5c67, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T13:25:01,075 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 3fb1b1f418508c88685394456e36ab0b in 534ms, sequenceid=37, compaction requested=false 2024-11-21T13:25:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:01,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-21T13:25:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-21T13:25:01,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-21T13:25:01,079 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 849 msec 2024-11-21T13:25:01,082 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 855 msec 2024-11-21T13:25:01,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:01,211 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:01,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T13:25:01,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/cc5c2eea274f4703ba7e1ec82a8a5cc2 is 50, key is test_row_0/A:col10/1732195501211/Put/seqid=0 2024-11-21T13:25:01,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195561240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195561241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,247 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195561245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195561247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195561248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,253 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742097_1273 (size=12001) 2024-11-21T13:25:01,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/cc5c2eea274f4703ba7e1ec82a8a5cc2 2024-11-21T13:25:01,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/10087a4f5bff4757ad0e60dcec815b7b is 50, key is test_row_0/B:col10/1732195501211/Put/seqid=0 2024-11-21T13:25:01,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742098_1274 (size=12001) 2024-11-21T13:25:01,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195561349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-21T13:25:01,363 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:01,363 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:01,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195561360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195561360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195561360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195561362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-21T13:25:01,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T13:25:01,366 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:01,367 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:01,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T13:25:01,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T13:25:01,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:01,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:01,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:01,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:01,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:01,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:01,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195561559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195561569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195561570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195561570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195561570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,573 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,673 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T13:25:01,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:01,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:01,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:01,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:01,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:01,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:01,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T13:25:01,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/10087a4f5bff4757ad0e60dcec815b7b 2024-11-21T13:25:01,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/3eeede3b66bc4005aba9ff319fb37092 is 50, key is test_row_0/C:col10/1732195501211/Put/seqid=0 2024-11-21T13:25:01,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742099_1275 (size=12001) 2024-11-21T13:25:01,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T13:25:01,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:01,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:01,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:01,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:01,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:01,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195561869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195561879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195561879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195561880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195561880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:01,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T13:25:01,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:01,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:01,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:01,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:01,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:01,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:01,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T13:25:02,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/3eeede3b66bc4005aba9ff319fb37092 2024-11-21T13:25:02,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/cc5c2eea274f4703ba7e1ec82a8a5cc2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/cc5c2eea274f4703ba7e1ec82a8a5cc2 2024-11-21T13:25:02,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/cc5c2eea274f4703ba7e1ec82a8a5cc2, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T13:25:02,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T13:25:02,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:02,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:02,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:02,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:02,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:02,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/10087a4f5bff4757ad0e60dcec815b7b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/10087a4f5bff4757ad0e60dcec815b7b 2024-11-21T13:25:02,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/10087a4f5bff4757ad0e60dcec815b7b, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T13:25:02,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/3eeede3b66bc4005aba9ff319fb37092 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/3eeede3b66bc4005aba9ff319fb37092 2024-11-21T13:25:02,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/3eeede3b66bc4005aba9ff319fb37092, entries=150, sequenceid=51, filesize=11.7 K 2024-11-21T13:25:02,157 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3fb1b1f418508c88685394456e36ab0b in 946ms, sequenceid=51, compaction requested=true 2024-11-21T13:25:02,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:02,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:02,158 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:02,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:02,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:02,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:02,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:02,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T13:25:02,158 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:02,161 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:02,161 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/A is initiating minor compaction (all files) 2024-11-21T13:25:02,161 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/A in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:02,161 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/ce0db2de02834cb5ae72b1010d94a484, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/caf1e6475d0c432086f041b9db2dad21, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/cc5c2eea274f4703ba7e1ec82a8a5cc2] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=39.7 K 2024-11-21T13:25:02,162 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:02,162 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/B is initiating minor compaction (all files) 2024-11-21T13:25:02,162 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/B in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:02,162 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/74a66cab894b47798422ba716040aad8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9da6a8f6337b4c7dbee0675037fe0e61, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/10087a4f5bff4757ad0e60dcec815b7b] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=35.2 K 2024-11-21T13:25:02,162 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce0db2de02834cb5ae72b1010d94a484, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732195500179 2024-11-21T13:25:02,163 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting caf1e6475d0c432086f041b9db2dad21, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732195500248 2024-11-21T13:25:02,163 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 74a66cab894b47798422ba716040aad8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732195500182 2024-11-21T13:25:02,164 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9da6a8f6337b4c7dbee0675037fe0e61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732195500248 2024-11-21T13:25:02,164 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc5c2eea274f4703ba7e1ec82a8a5cc2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732195501210 2024-11-21T13:25:02,165 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 10087a4f5bff4757ad0e60dcec815b7b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732195501210 2024-11-21T13:25:02,192 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#A#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:02,193 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/921b990c38cb4b919b76f134b018a1eb is 50, key is test_row_0/A:col10/1732195501211/Put/seqid=0 2024-11-21T13:25:02,202 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#B#compaction#222 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:02,203 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/1551f596220747fa9cb425c594e54d4c is 50, key is test_row_0/B:col10/1732195501211/Put/seqid=0 2024-11-21T13:25:02,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742100_1276 (size=12104) 2024-11-21T13:25:02,253 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/921b990c38cb4b919b76f134b018a1eb as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/921b990c38cb4b919b76f134b018a1eb 2024-11-21T13:25:02,265 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/A of 3fb1b1f418508c88685394456e36ab0b into 921b990c38cb4b919b76f134b018a1eb(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:02,265 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:02,265 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/A, priority=13, startTime=1732195502158; duration=0sec 2024-11-21T13:25:02,265 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:02,265 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:A 2024-11-21T13:25:02,265 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:02,268 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:02,269 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/C is initiating minor compaction (all files) 2024-11-21T13:25:02,269 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/C in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:02,269 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/819411b05efd401182dab0dcfd69b54a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/10e6a4a13f494613b607f4f75b2e5c67, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/3eeede3b66bc4005aba9ff319fb37092] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=35.2 K 2024-11-21T13:25:02,269 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 819411b05efd401182dab0dcfd69b54a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732195500182 2024-11-21T13:25:02,270 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10e6a4a13f494613b607f4f75b2e5c67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732195500248 2024-11-21T13:25:02,270 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eeede3b66bc4005aba9ff319fb37092, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732195501210 2024-11-21T13:25:02,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742101_1277 (size=12104) 2024-11-21T13:25:02,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-21T13:25:02,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:02,288 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T13:25:02,297 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/1551f596220747fa9cb425c594e54d4c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/1551f596220747fa9cb425c594e54d4c 2024-11-21T13:25:02,303 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/B of 3fb1b1f418508c88685394456e36ab0b into 1551f596220747fa9cb425c594e54d4c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:02,303 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:02,303 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/B, priority=13, startTime=1732195502158; duration=0sec 2024-11-21T13:25:02,303 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:02,303 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:B 2024-11-21T13:25:02,304 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#C#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:02,311 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/f55e084c0df949dbbbcf0c076bc04b40 is 50, key is test_row_0/C:col10/1732195501211/Put/seqid=0 2024-11-21T13:25:02,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/447827fb577b49debb5e0fec26506a61 is 50, key is test_row_0/A:col10/1732195501239/Put/seqid=0 2024-11-21T13:25:02,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742102_1278 (size=12104) 2024-11-21T13:25:02,379 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/f55e084c0df949dbbbcf0c076bc04b40 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/f55e084c0df949dbbbcf0c076bc04b40 2024-11-21T13:25:02,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:02,388 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/C of 3fb1b1f418508c88685394456e36ab0b into f55e084c0df949dbbbcf0c076bc04b40(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:02,389 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:02,389 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/C, priority=13, startTime=1732195502158; duration=0sec 2024-11-21T13:25:02,389 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:02,389 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:C 2024-11-21T13:25:02,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742103_1279 (size=12001) 2024-11-21T13:25:02,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195562401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195562401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195562402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195562402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195562402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T13:25:02,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195562509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195562509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195562519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195562519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195562520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,523 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195562787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195562787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195562788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195562788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,793 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195562789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:02,793 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:02,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:02,798 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/447827fb577b49debb5e0fec26506a61 2024-11-21T13:25:02,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/c459800bc7354c17abbe4c9315f2e1f7 is 50, key is test_row_0/B:col10/1732195501239/Put/seqid=0 2024-11-21T13:25:02,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742104_1280 (size=12001) 2024-11-21T13:25:02,821 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/c459800bc7354c17abbe4c9315f2e1f7 2024-11-21T13:25:02,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/599bce3fcaea4be389dbfddaa3dd461c is 50, key is test_row_0/C:col10/1732195501239/Put/seqid=0 2024-11-21T13:25:02,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742105_1281 (size=12001) 2024-11-21T13:25:03,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195563099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195563099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195563099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195563099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195563099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,234 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/599bce3fcaea4be389dbfddaa3dd461c 2024-11-21T13:25:03,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/447827fb577b49debb5e0fec26506a61 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/447827fb577b49debb5e0fec26506a61 2024-11-21T13:25:03,245 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/447827fb577b49debb5e0fec26506a61, entries=150, sequenceid=74, filesize=11.7 K 2024-11-21T13:25:03,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/c459800bc7354c17abbe4c9315f2e1f7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/c459800bc7354c17abbe4c9315f2e1f7 2024-11-21T13:25:03,250 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/c459800bc7354c17abbe4c9315f2e1f7, entries=150, sequenceid=74, filesize=11.7 K 2024-11-21T13:25:03,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/599bce3fcaea4be389dbfddaa3dd461c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/599bce3fcaea4be389dbfddaa3dd461c 2024-11-21T13:25:03,256 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/599bce3fcaea4be389dbfddaa3dd461c, entries=150, sequenceid=74, filesize=11.7 K 2024-11-21T13:25:03,257 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=80.51 KB/82440 for 3fb1b1f418508c88685394456e36ab0b in 969ms, sequenceid=74, compaction requested=false 2024-11-21T13:25:03,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:03,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:03,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-21T13:25:03,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-21T13:25:03,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-21T13:25:03,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8920 sec 2024-11-21T13:25:03,263 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 1.8990 sec 2024-11-21T13:25:03,325 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T13:25:03,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-21T13:25:03,509 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:03,510 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:03,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-21T13:25:03,512 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:03,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T13:25:03,512 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:03,513 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:03,612 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:03,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T13:25:03,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/3f9d8459846a42e789cc2d0d17154153 is 50, key is test_row_0/A:col10/1732195503611/Put/seqid=0 2024-11-21T13:25:03,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T13:25:03,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742106_1282 (size=12001) 2024-11-21T13:25:03,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195563627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195563627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195563628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,633 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195563629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195563631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T13:25:03,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:03,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:03,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:03,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:03,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:03,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195563739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195563739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195563739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195563739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195563739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,745 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,819 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T13:25:03,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:03,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:03,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:03,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T13:25:03,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195563949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195563950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,952 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195563950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,953 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195563950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195563950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:03,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:03,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T13:25:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:03,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:03,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:03,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,022 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/3f9d8459846a42e789cc2d0d17154153 2024-11-21T13:25:04,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/46f0ed1fa6b5437b89c04a84239a0d62 is 50, key is test_row_0/B:col10/1732195503611/Put/seqid=0 2024-11-21T13:25:04,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742107_1283 (size=12001) 2024-11-21T13:25:04,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T13:25:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:04,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:04,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T13:25:04,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:04,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195564259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195564259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:04,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195564259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:04,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195564259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:04,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195564259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,262 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:04,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T13:25:04,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:04,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:04,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:04,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T13:25:04,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:04,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:04,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:04,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/46f0ed1fa6b5437b89c04a84239a0d62 2024-11-21T13:25:04,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/481b4ada78f0439aa4f74bd16c5e00be is 50, key is test_row_0/C:col10/1732195503611/Put/seqid=0 2024-11-21T13:25:04,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742108_1284 (size=12001) 2024-11-21T13:25:04,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T13:25:04,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:04,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:04,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:04,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T13:25:04,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T13:25:04,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:04,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:04,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:04,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:04,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:04,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195564769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:04,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:04,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195564770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:04,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:04,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195564770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:04,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195564770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:04,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:04,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195564770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:04,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:04,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/481b4ada78f0439aa4f74bd16c5e00be 2024-11-21T13:25:04,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/3f9d8459846a42e789cc2d0d17154153 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/3f9d8459846a42e789cc2d0d17154153 2024-11-21T13:25:04,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/3f9d8459846a42e789cc2d0d17154153, entries=150, sequenceid=93, filesize=11.7 K 2024-11-21T13:25:04,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/46f0ed1fa6b5437b89c04a84239a0d62 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/46f0ed1fa6b5437b89c04a84239a0d62 2024-11-21T13:25:04,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/46f0ed1fa6b5437b89c04a84239a0d62, entries=150, sequenceid=93, filesize=11.7 K 2024-11-21T13:25:04,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/481b4ada78f0439aa4f74bd16c5e00be as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/481b4ada78f0439aa4f74bd16c5e00be 2024-11-21T13:25:04,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/481b4ada78f0439aa4f74bd16c5e00be, entries=150, sequenceid=93, filesize=11.7 K 2024-11-21T13:25:04,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 3fb1b1f418508c88685394456e36ab0b in 1278ms, sequenceid=93, compaction requested=true 2024-11-21T13:25:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:04,890 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:04,890 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:04,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:04,891 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:04,891 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:04,891 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/B is initiating minor compaction (all files) 2024-11-21T13:25:04,891 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/A is initiating minor compaction (all files) 2024-11-21T13:25:04,891 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/B in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:04,891 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/A in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:04,891 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/921b990c38cb4b919b76f134b018a1eb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/447827fb577b49debb5e0fec26506a61, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/3f9d8459846a42e789cc2d0d17154153] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=35.3 K 2024-11-21T13:25:04,891 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/1551f596220747fa9cb425c594e54d4c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/c459800bc7354c17abbe4c9315f2e1f7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/46f0ed1fa6b5437b89c04a84239a0d62] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=35.3 K 2024-11-21T13:25:04,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-21T13:25:04,892 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 1551f596220747fa9cb425c594e54d4c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732195501210 2024-11-21T13:25:04,892 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 921b990c38cb4b919b76f134b018a1eb, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732195501210 2024-11-21T13:25:04,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:04,892 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T13:25:04,894 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting c459800bc7354c17abbe4c9315f2e1f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732195501239 2024-11-21T13:25:04,894 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 447827fb577b49debb5e0fec26506a61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732195501239 2024-11-21T13:25:04,895 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f9d8459846a42e789cc2d0d17154153, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732195502401 2024-11-21T13:25:04,895 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 46f0ed1fa6b5437b89c04a84239a0d62, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732195502401 2024-11-21T13:25:04,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/16862ba94d294226a98e13ec13abcec2 is 50, key is test_row_0/A:col10/1732195503629/Put/seqid=0 2024-11-21T13:25:04,934 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#A#compaction#231 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:04,935 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/40558743d8e2497e991f9df968888af6 is 50, key is test_row_0/A:col10/1732195503611/Put/seqid=0 2024-11-21T13:25:04,936 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#B#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:04,937 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/dc46f6b2d02b4def98df00aa8f047e54 is 50, key is test_row_0/B:col10/1732195503611/Put/seqid=0 2024-11-21T13:25:04,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742109_1285 (size=12001) 2024-11-21T13:25:04,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742110_1286 (size=12207) 2024-11-21T13:25:04,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742111_1287 (size=12207) 2024-11-21T13:25:05,343 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/16862ba94d294226a98e13ec13abcec2 2024-11-21T13:25:05,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/95641fb7f3b04f31bda16033ce526b0a is 50, key is test_row_0/B:col10/1732195503629/Put/seqid=0 2024-11-21T13:25:05,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742112_1288 (size=12001) 2024-11-21T13:25:05,363 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/40558743d8e2497e991f9df968888af6 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/40558743d8e2497e991f9df968888af6 2024-11-21T13:25:05,363 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/95641fb7f3b04f31bda16033ce526b0a 2024-11-21T13:25:05,382 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/dc46f6b2d02b4def98df00aa8f047e54 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/dc46f6b2d02b4def98df00aa8f047e54 2024-11-21T13:25:05,394 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/A of 3fb1b1f418508c88685394456e36ab0b into 40558743d8e2497e991f9df968888af6(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:05,394 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:05,394 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/A, priority=13, startTime=1732195504890; duration=0sec 2024-11-21T13:25:05,394 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:05,394 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:A 2024-11-21T13:25:05,394 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:05,397 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:05,397 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/C is initiating minor compaction (all files) 2024-11-21T13:25:05,397 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/C in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:05,397 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/f55e084c0df949dbbbcf0c076bc04b40, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/599bce3fcaea4be389dbfddaa3dd461c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/481b4ada78f0439aa4f74bd16c5e00be] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=35.3 K 2024-11-21T13:25:05,397 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting f55e084c0df949dbbbcf0c076bc04b40, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1732195501210 2024-11-21T13:25:05,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/6f86adee467c421e9c7dd29719c2784f is 50, key is test_row_0/C:col10/1732195503629/Put/seqid=0 2024-11-21T13:25:05,398 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 599bce3fcaea4be389dbfddaa3dd461c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732195501239 2024-11-21T13:25:05,398 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 481b4ada78f0439aa4f74bd16c5e00be, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732195502401 2024-11-21T13:25:05,398 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/B of 3fb1b1f418508c88685394456e36ab0b into dc46f6b2d02b4def98df00aa8f047e54(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:05,398 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:05,399 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/B, priority=13, startTime=1732195504890; duration=0sec 2024-11-21T13:25:05,399 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:05,399 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:B 2024-11-21T13:25:05,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742113_1289 (size=12001) 2024-11-21T13:25:05,418 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#C#compaction#235 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:05,418 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/6f86adee467c421e9c7dd29719c2784f 2024-11-21T13:25:05,419 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/2f44612a05134d43bacc433606fde55e is 50, key is test_row_0/C:col10/1732195503611/Put/seqid=0 2024-11-21T13:25:05,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/16862ba94d294226a98e13ec13abcec2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/16862ba94d294226a98e13ec13abcec2 2024-11-21T13:25:05,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742114_1290 (size=12207) 2024-11-21T13:25:05,432 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/16862ba94d294226a98e13ec13abcec2, entries=150, sequenceid=112, filesize=11.7 K 2024-11-21T13:25:05,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/95641fb7f3b04f31bda16033ce526b0a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/95641fb7f3b04f31bda16033ce526b0a 2024-11-21T13:25:05,442 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/95641fb7f3b04f31bda16033ce526b0a, entries=150, sequenceid=112, filesize=11.7 K 2024-11-21T13:25:05,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/6f86adee467c421e9c7dd29719c2784f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/6f86adee467c421e9c7dd29719c2784f 2024-11-21T13:25:05,453 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/6f86adee467c421e9c7dd29719c2784f, entries=150, sequenceid=112, filesize=11.7 K 2024-11-21T13:25:05,455 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for 3fb1b1f418508c88685394456e36ab0b in 562ms, sequenceid=112, compaction requested=false 2024-11-21T13:25:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:05,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-21T13:25:05,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-21T13:25:05,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-21T13:25:05,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9430 sec 2024-11-21T13:25:05,460 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.9490 sec 2024-11-21T13:25:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-21T13:25:05,671 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:05,671 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:05,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-21T13:25:05,674 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:05,675 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:05,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:05,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T13:25:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T13:25:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:05,792 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:05,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:25:05,798 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/68305233b0ca4f428e94a2652b66fac8 is 50, key is test_row_0/A:col10/1732195505788/Put/seqid=0 2024-11-21T13:25:05,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742115_1291 (size=12001) 2024-11-21T13:25:05,816 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/68305233b0ca4f428e94a2652b66fac8 2024-11-21T13:25:05,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/e17ccd236205402fadce1423e6664f1c is 50, key is test_row_0/B:col10/1732195505788/Put/seqid=0 2024-11-21T13:25:05,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742116_1292 (size=12001) 2024-11-21T13:25:05,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T13:25:05,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:05,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:05,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:05,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:05,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:05,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:05,834 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/2f44612a05134d43bacc433606fde55e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/2f44612a05134d43bacc433606fde55e 2024-11-21T13:25:05,854 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/C of 3fb1b1f418508c88685394456e36ab0b into 2f44612a05134d43bacc433606fde55e(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:05,854 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:05,855 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/C, priority=13, startTime=1732195504890; duration=0sec 2024-11-21T13:25:05,855 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:05,855 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:C 2024-11-21T13:25:05,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:05,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195565854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:05,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:05,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195565861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,868 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:05,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195565866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:05,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:05,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195565869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:05,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195565869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195565979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195565979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195565979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:05,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T13:25:05,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:05,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:05,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195565979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:05,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:05,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195565979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:05,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:05,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:05,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:05,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T13:25:06,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T13:25:06,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:06,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:06,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:06,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:06,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:06,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:06,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:06,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195566189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:06,190 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195566189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195566189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195566190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:06,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195566190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:06,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/e17ccd236205402fadce1423e6664f1c 2024-11-21T13:25:06,238 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/828b0af726924a2fbf2c7057657ba0a9 is 50, key is test_row_0/C:col10/1732195505788/Put/seqid=0 2024-11-21T13:25:06,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742117_1293 (size=12001) 2024-11-21T13:25:06,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T13:25:06,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:06,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:06,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:06,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:06,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:06,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T13:25:06,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T13:25:06,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:06,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:06,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:06,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:06,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:06,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195566499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,501 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:06,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195566499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195566499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:06,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195566499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:06,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:06,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195566500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:06,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T13:25:06,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:06,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:06,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:06,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:06,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:06,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:06,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/828b0af726924a2fbf2c7057657ba0a9 2024-11-21T13:25:06,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/68305233b0ca4f428e94a2652b66fac8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/68305233b0ca4f428e94a2652b66fac8 2024-11-21T13:25:06,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/68305233b0ca4f428e94a2652b66fac8, entries=150, sequenceid=125, filesize=11.7 K 2024-11-21T13:25:06,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/e17ccd236205402fadce1423e6664f1c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/e17ccd236205402fadce1423e6664f1c 2024-11-21T13:25:06,665 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/e17ccd236205402fadce1423e6664f1c, entries=150, sequenceid=125, filesize=11.7 K 2024-11-21T13:25:06,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/828b0af726924a2fbf2c7057657ba0a9 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/828b0af726924a2fbf2c7057657ba0a9 2024-11-21T13:25:06,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/828b0af726924a2fbf2c7057657ba0a9, entries=150, sequenceid=125, filesize=11.7 K 2024-11-21T13:25:06,675 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 3fb1b1f418508c88685394456e36ab0b in 883ms, sequenceid=125, compaction requested=true 2024-11-21T13:25:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:06,675 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:06,675 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:06,675 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:06,677 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:06,677 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:06,677 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/A is initiating minor compaction (all files) 2024-11-21T13:25:06,677 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/A in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:06,677 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/40558743d8e2497e991f9df968888af6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/16862ba94d294226a98e13ec13abcec2, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/68305233b0ca4f428e94a2652b66fac8] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=35.4 K 2024-11-21T13:25:06,677 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/B is initiating minor compaction (all files) 2024-11-21T13:25:06,677 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/B in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:06,677 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/dc46f6b2d02b4def98df00aa8f047e54, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/95641fb7f3b04f31bda16033ce526b0a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/e17ccd236205402fadce1423e6664f1c] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=35.4 K 2024-11-21T13:25:06,678 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting dc46f6b2d02b4def98df00aa8f047e54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732195502401 2024-11-21T13:25:06,678 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40558743d8e2497e991f9df968888af6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732195502401 2024-11-21T13:25:06,678 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 95641fb7f3b04f31bda16033ce526b0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732195503626 2024-11-21T13:25:06,678 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16862ba94d294226a98e13ec13abcec2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732195503626 2024-11-21T13:25:06,679 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e17ccd236205402fadce1423e6664f1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732195505788 2024-11-21T13:25:06,679 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68305233b0ca4f428e94a2652b66fac8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732195505788 2024-11-21T13:25:06,690 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#B#compaction#239 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:06,691 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/7191f46413274c27b3d2dc8b6fa5b854 is 50, key is test_row_0/B:col10/1732195505788/Put/seqid=0 2024-11-21T13:25:06,694 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:06,694 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/6923732cccc646c1b933c190f5a0e5f5 is 50, key is test_row_0/A:col10/1732195505788/Put/seqid=0 2024-11-21T13:25:06,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742118_1294 (size=12309) 2024-11-21T13:25:06,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742119_1295 (size=12309) 2024-11-21T13:25:06,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-21T13:25:06,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:06,750 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T13:25:06,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/8079fea0de2a4166b6a7c0616d9a4ebf is 50, key is test_row_0/A:col10/1732195505865/Put/seqid=0 2024-11-21T13:25:06,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742120_1296 (size=12151) 2024-11-21T13:25:06,762 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/8079fea0de2a4166b6a7c0616d9a4ebf 2024-11-21T13:25:06,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/5e8fe6c8555d412fa82de61c7a150374 is 50, key is test_row_0/B:col10/1732195505865/Put/seqid=0 2024-11-21T13:25:06,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742121_1297 (size=12151) 2024-11-21T13:25:06,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T13:25:07,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:07,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:07,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195567016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195567019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195567021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195567024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195567024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,124 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/6923732cccc646c1b933c190f5a0e5f5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/6923732cccc646c1b933c190f5a0e5f5 2024-11-21T13:25:07,126 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/7191f46413274c27b3d2dc8b6fa5b854 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/7191f46413274c27b3d2dc8b6fa5b854 2024-11-21T13:25:07,132 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/B of 3fb1b1f418508c88685394456e36ab0b into 7191f46413274c27b3d2dc8b6fa5b854(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:07,132 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:07,132 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/B, priority=13, startTime=1732195506675; duration=0sec 2024-11-21T13:25:07,132 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:07,132 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:B 2024-11-21T13:25:07,132 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:07,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195567129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195567129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,133 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,134 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:07,135 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/C is initiating minor compaction (all files) 2024-11-21T13:25:07,135 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/C in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:07,135 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/2f44612a05134d43bacc433606fde55e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/6f86adee467c421e9c7dd29719c2784f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/828b0af726924a2fbf2c7057657ba0a9] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=35.4 K 2024-11-21T13:25:07,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195567133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,136 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f44612a05134d43bacc433606fde55e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1732195502401 2024-11-21T13:25:07,136 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f86adee467c421e9c7dd29719c2784f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1732195503626 2024-11-21T13:25:07,137 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 828b0af726924a2fbf2c7057657ba0a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732195505788 2024-11-21T13:25:07,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195567139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,141 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/A of 3fb1b1f418508c88685394456e36ab0b into 6923732cccc646c1b933c190f5a0e5f5(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:07,141 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:07,141 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/A, priority=13, startTime=1732195506675; duration=0sec 2024-11-21T13:25:07,142 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:07,142 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:A 2024-11-21T13:25:07,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195567140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,152 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#C#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:07,153 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/b85e769a31184bd896dd22f349d31d61 is 50, key is test_row_0/C:col10/1732195505788/Put/seqid=0 2024-11-21T13:25:07,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742122_1298 (size=12309) 2024-11-21T13:25:07,183 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/5e8fe6c8555d412fa82de61c7a150374 2024-11-21T13:25:07,183 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/b85e769a31184bd896dd22f349d31d61 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b85e769a31184bd896dd22f349d31d61 2024-11-21T13:25:07,194 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/C of 3fb1b1f418508c88685394456e36ab0b into b85e769a31184bd896dd22f349d31d61(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:07,194 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:07,194 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/C, priority=13, startTime=1732195506675; duration=0sec 2024-11-21T13:25:07,194 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:07,194 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:C 2024-11-21T13:25:07,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/2648c34f58b04c15a0811153135b7a75 is 50, key is test_row_0/C:col10/1732195505865/Put/seqid=0 2024-11-21T13:25:07,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742123_1299 (size=12151) 2024-11-21T13:25:07,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195567339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195567339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195567339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195567349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195567349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,630 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/2648c34f58b04c15a0811153135b7a75 2024-11-21T13:25:07,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/8079fea0de2a4166b6a7c0616d9a4ebf as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/8079fea0de2a4166b6a7c0616d9a4ebf 2024-11-21T13:25:07,644 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/8079fea0de2a4166b6a7c0616d9a4ebf, entries=150, sequenceid=152, filesize=11.9 K 2024-11-21T13:25:07,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/5e8fe6c8555d412fa82de61c7a150374 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/5e8fe6c8555d412fa82de61c7a150374 2024-11-21T13:25:07,652 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/5e8fe6c8555d412fa82de61c7a150374, entries=150, sequenceid=152, filesize=11.9 K 2024-11-21T13:25:07,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195567649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/2648c34f58b04c15a0811153135b7a75 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/2648c34f58b04c15a0811153135b7a75 2024-11-21T13:25:07,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195567649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195567649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,659 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/2648c34f58b04c15a0811153135b7a75, entries=150, sequenceid=152, filesize=11.9 K 2024-11-21T13:25:07,661 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 3fb1b1f418508c88685394456e36ab0b in 912ms, sequenceid=152, compaction requested=false 2024-11-21T13:25:07,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:07,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:07,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-21T13:25:07,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-21T13:25:07,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:07,662 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:07,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T13:25:07,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-21T13:25:07,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9880 sec 2024-11-21T13:25:07,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/1daf35cb015f4a95a30972f6fe723540 is 50, key is test_row_0/A:col10/1732195507021/Put/seqid=0 2024-11-21T13:25:07,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.9960 sec 2024-11-21T13:25:07,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742124_1300 (size=16931) 2024-11-21T13:25:07,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/1daf35cb015f4a95a30972f6fe723540 2024-11-21T13:25:07,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/9ea2fdcd16a04b2d9edd47bc52a3fb18 is 50, key is test_row_0/B:col10/1732195507021/Put/seqid=0 2024-11-21T13:25:07,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195567731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195567735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742125_1301 (size=12151) 2024-11-21T13:25:07,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-21T13:25:07,820 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:07,820 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:07,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-21T13:25:07,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T13:25:07,824 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:07,826 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:07,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:07,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195567839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,841 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:07,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195567849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:07,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:07,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T13:25:07,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T13:25:07,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:07,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:07,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:07,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:07,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:07,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:08,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:08,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195568049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:08,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:08,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:08,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195568059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:08,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T13:25:08,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:08,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:08,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:08,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:08,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:08,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:08,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T13:25:08,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/9ea2fdcd16a04b2d9edd47bc52a3fb18 2024-11-21T13:25:08,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/f4ad98108dd94fd6b642bf43ace39fae is 50, key is test_row_0/C:col10/1732195507021/Put/seqid=0 2024-11-21T13:25:08,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742126_1302 (size=12151) 2024-11-21T13:25:08,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195568159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:08,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:08,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195568159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:08,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:08,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:08,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:08,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195568169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:08,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:08,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T13:25:08,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:08,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:08,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:08,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:08,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:08,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:08,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195568359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:08,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:08,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195568369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:08,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T13:25:08,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:08,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:08,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:08,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:08,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:08,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T13:25:08,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/f4ad98108dd94fd6b642bf43ace39fae 2024-11-21T13:25:08,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/1daf35cb015f4a95a30972f6fe723540 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/1daf35cb015f4a95a30972f6fe723540 2024-11-21T13:25:08,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/1daf35cb015f4a95a30972f6fe723540, entries=250, sequenceid=167, filesize=16.5 K 2024-11-21T13:25:08,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/9ea2fdcd16a04b2d9edd47bc52a3fb18 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9ea2fdcd16a04b2d9edd47bc52a3fb18 2024-11-21T13:25:08,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9ea2fdcd16a04b2d9edd47bc52a3fb18, entries=150, sequenceid=167, filesize=11.9 K 2024-11-21T13:25:08,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/f4ad98108dd94fd6b642bf43ace39fae as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/f4ad98108dd94fd6b642bf43ace39fae 2024-11-21T13:25:08,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/f4ad98108dd94fd6b642bf43ace39fae, entries=150, sequenceid=167, filesize=11.9 K 2024-11-21T13:25:08,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 3fb1b1f418508c88685394456e36ab0b in 915ms, sequenceid=167, compaction requested=true 2024-11-21T13:25:08,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:08,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:08,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:08,578 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:08,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:08,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:08,578 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:08,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:08,578 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:08,579 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41391 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:08,579 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/A is initiating minor compaction (all files) 2024-11-21T13:25:08,579 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:08,579 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/A in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:08,579 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/B is initiating minor compaction (all files) 2024-11-21T13:25:08,579 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/6923732cccc646c1b933c190f5a0e5f5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/8079fea0de2a4166b6a7c0616d9a4ebf, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/1daf35cb015f4a95a30972f6fe723540] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=40.4 K 2024-11-21T13:25:08,579 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/B in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:08,579 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/7191f46413274c27b3d2dc8b6fa5b854, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/5e8fe6c8555d412fa82de61c7a150374, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9ea2fdcd16a04b2d9edd47bc52a3fb18] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=35.8 K 2024-11-21T13:25:08,579 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6923732cccc646c1b933c190f5a0e5f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732195505788 2024-11-21T13:25:08,579 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 7191f46413274c27b3d2dc8b6fa5b854, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732195505788 2024-11-21T13:25:08,580 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8079fea0de2a4166b6a7c0616d9a4ebf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732195505859 2024-11-21T13:25:08,580 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e8fe6c8555d412fa82de61c7a150374, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732195505859 2024-11-21T13:25:08,580 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1daf35cb015f4a95a30972f6fe723540, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732195507012 2024-11-21T13:25:08,580 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ea2fdcd16a04b2d9edd47bc52a3fb18, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732195507021 2024-11-21T13:25:08,587 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#B#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:08,587 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/145ab3e01bc943708a0b7920c2379e8d is 50, key is test_row_0/B:col10/1732195507021/Put/seqid=0 2024-11-21T13:25:08,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-21T13:25:08,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:08,592 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T13:25:08,602 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#A#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:08,603 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/11137f04db3645f1ad4417d702385f4c is 50, key is test_row_0/A:col10/1732195507021/Put/seqid=0 2024-11-21T13:25:08,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/52957e4e35144d3ebcad805637f6c7d3 is 50, key is test_row_0/A:col10/1732195507721/Put/seqid=0 2024-11-21T13:25:08,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742127_1303 (size=12561) 2024-11-21T13:25:08,636 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/145ab3e01bc943708a0b7920c2379e8d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/145ab3e01bc943708a0b7920c2379e8d 2024-11-21T13:25:08,642 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/B of 3fb1b1f418508c88685394456e36ab0b into 145ab3e01bc943708a0b7920c2379e8d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:08,642 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:08,642 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/B, priority=13, startTime=1732195508578; duration=0sec 2024-11-21T13:25:08,642 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:08,642 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:B 2024-11-21T13:25:08,642 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:08,644 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:08,644 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/C is initiating minor compaction (all files) 2024-11-21T13:25:08,644 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/C in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:08,644 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b85e769a31184bd896dd22f349d31d61, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/2648c34f58b04c15a0811153135b7a75, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/f4ad98108dd94fd6b642bf43ace39fae] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=35.8 K 2024-11-21T13:25:08,644 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b85e769a31184bd896dd22f349d31d61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732195505788 2024-11-21T13:25:08,645 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 2648c34f58b04c15a0811153135b7a75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1732195505859 2024-11-21T13:25:08,646 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f4ad98108dd94fd6b642bf43ace39fae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732195507021 2024-11-21T13:25:08,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742129_1305 (size=12151) 2024-11-21T13:25:08,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742128_1304 (size=12561) 2024-11-21T13:25:08,690 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:08,691 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/c89dd676f6a746e5b57b6b1ff25c511c is 50, key is test_row_0/C:col10/1732195507021/Put/seqid=0 2024-11-21T13:25:08,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742130_1306 (size=12561) 2024-11-21T13:25:08,704 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/c89dd676f6a746e5b57b6b1ff25c511c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/c89dd676f6a746e5b57b6b1ff25c511c 2024-11-21T13:25:08,710 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/C of 3fb1b1f418508c88685394456e36ab0b into c89dd676f6a746e5b57b6b1ff25c511c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:08,710 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:08,710 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/C, priority=13, startTime=1732195508578; duration=0sec 2024-11-21T13:25:08,711 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:08,711 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:C 2024-11-21T13:25:08,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:08,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:08,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:08,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195568889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:08,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:08,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195568889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:08,893 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:08,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T13:25:09,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:09,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195568999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:09,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195568999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,001 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:09,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:09,062 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/52957e4e35144d3ebcad805637f6c7d3 2024-11-21T13:25:09,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/3bcb670f499f4787a76c78490891250d is 50, key is test_row_0/B:col10/1732195507721/Put/seqid=0 2024-11-21T13:25:09,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742131_1307 (size=12151) 2024-11-21T13:25:09,086 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/11137f04db3645f1ad4417d702385f4c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/11137f04db3645f1ad4417d702385f4c 2024-11-21T13:25:09,089 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/A of 3fb1b1f418508c88685394456e36ab0b into 11137f04db3645f1ad4417d702385f4c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:09,089 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:09,089 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/A, priority=13, startTime=1732195508578; duration=0sec 2024-11-21T13:25:09,090 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:09,090 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:A 2024-11-21T13:25:09,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:09,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195569169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:09,171 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 3fb1b1f418508c88685394456e36ab0b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:25:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:09,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:09,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195569179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:09,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:09,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195569179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:09,181 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_1' in 3fb1b1f418508c88685394456e36ab0b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:25:09,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:09,181 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_1' in 3fb1b1f418508c88685394456e36ab0b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:25:09,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:09,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195569209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:09,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195569209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:09,472 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/3bcb670f499f4787a76c78490891250d 2024-11-21T13:25:09,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/29ffe480d35c401994863788240479f0 is 50, key is test_row_0/C:col10/1732195507721/Put/seqid=0 2024-11-21T13:25:09,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742132_1308 (size=12151) 2024-11-21T13:25:09,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195569519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:09,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:09,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195569519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:09,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:09,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:09,882 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/29ffe480d35c401994863788240479f0 2024-11-21T13:25:09,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/52957e4e35144d3ebcad805637f6c7d3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/52957e4e35144d3ebcad805637f6c7d3 2024-11-21T13:25:09,889 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/52957e4e35144d3ebcad805637f6c7d3, entries=150, sequenceid=191, filesize=11.9 K 2024-11-21T13:25:09,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/3bcb670f499f4787a76c78490891250d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/3bcb670f499f4787a76c78490891250d 2024-11-21T13:25:09,893 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/3bcb670f499f4787a76c78490891250d, entries=150, sequenceid=191, filesize=11.9 K 2024-11-21T13:25:09,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/29ffe480d35c401994863788240479f0 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/29ffe480d35c401994863788240479f0 2024-11-21T13:25:09,897 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/29ffe480d35c401994863788240479f0, entries=150, sequenceid=191, filesize=11.9 K 2024-11-21T13:25:09,898 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 3fb1b1f418508c88685394456e36ab0b in 1307ms, sequenceid=191, compaction requested=false 2024-11-21T13:25:09,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:09,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:09,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-21T13:25:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-21T13:25:09,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-21T13:25:09,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0720 sec 2024-11-21T13:25:09,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.0810 sec 2024-11-21T13:25:09,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-21T13:25:09,969 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:09,969 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:09,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-21T13:25:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T13:25:09,971 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:09,972 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:09,972 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:10,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:10,032 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:10,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T13:25:10,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/749910f0e34f40c695a8672c8740be1c is 50, key is test_row_0/A:col10/1732195510031/Put/seqid=0 2024-11-21T13:25:10,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742133_1309 (size=16931) 2024-11-21T13:25:10,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:10,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195570069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:10,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,072 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:10,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:10,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195570072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,075 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:10,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T13:25:10,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T13:25:10,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:10,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195570179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:10,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195570179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:10,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:10,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T13:25:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,276 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T13:25:10,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195570389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:10,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195570389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,390 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,391 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:10,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T13:25:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,429 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/749910f0e34f40c695a8672c8740be1c 2024-11-21T13:25:10,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/575701ee8b0d4b3d8317b5053bcced55 is 50, key is test_row_0/B:col10/1732195510031/Put/seqid=0 2024-11-21T13:25:10,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742134_1310 (size=12151) 2024-11-21T13:25:10,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/575701ee8b0d4b3d8317b5053bcced55 2024-11-21T13:25:10,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/b5dbc203fccf49aca97b7bd576fa048e is 50, key is test_row_0/C:col10/1732195510031/Put/seqid=0 2024-11-21T13:25:10,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742135_1311 (size=12151) 2024-11-21T13:25:10,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T13:25:10,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:10,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T13:25:10,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195570699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:10,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195570699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:10,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:10,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T13:25:10,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:10,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:10,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/b5dbc203fccf49aca97b7bd576fa048e 2024-11-21T13:25:10,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/749910f0e34f40c695a8672c8740be1c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/749910f0e34f40c695a8672c8740be1c 2024-11-21T13:25:10,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/749910f0e34f40c695a8672c8740be1c, entries=250, sequenceid=208, filesize=16.5 K 2024-11-21T13:25:10,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/575701ee8b0d4b3d8317b5053bcced55 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/575701ee8b0d4b3d8317b5053bcced55 2024-11-21T13:25:10,874 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/575701ee8b0d4b3d8317b5053bcced55, entries=150, sequenceid=208, filesize=11.9 K 2024-11-21T13:25:10,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/b5dbc203fccf49aca97b7bd576fa048e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b5dbc203fccf49aca97b7bd576fa048e 2024-11-21T13:25:10,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b5dbc203fccf49aca97b7bd576fa048e, entries=150, sequenceid=208, filesize=11.9 K 2024-11-21T13:25:10,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 3fb1b1f418508c88685394456e36ab0b in 847ms, sequenceid=208, compaction requested=true 2024-11-21T13:25:10,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:10,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:10,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:10,879 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:10,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:10,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:10,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:10,880 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:10,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:10,880 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:10,880 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:10,880 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/B is initiating minor compaction (all files) 2024-11-21T13:25:10,880 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/A is initiating minor compaction (all files) 2024-11-21T13:25:10,880 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/B in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,880 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/A in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,880 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/145ab3e01bc943708a0b7920c2379e8d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/3bcb670f499f4787a76c78490891250d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/575701ee8b0d4b3d8317b5053bcced55] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.0 K 2024-11-21T13:25:10,880 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/11137f04db3645f1ad4417d702385f4c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/52957e4e35144d3ebcad805637f6c7d3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/749910f0e34f40c695a8672c8740be1c] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=40.7 K 2024-11-21T13:25:10,881 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 145ab3e01bc943708a0b7920c2379e8d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732195507021 2024-11-21T13:25:10,881 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11137f04db3645f1ad4417d702385f4c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732195507021 2024-11-21T13:25:10,881 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bcb670f499f4787a76c78490891250d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732195507709 2024-11-21T13:25:10,881 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52957e4e35144d3ebcad805637f6c7d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732195507709 2024-11-21T13:25:10,881 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 575701ee8b0d4b3d8317b5053bcced55, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732195508880 2024-11-21T13:25:10,882 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 749910f0e34f40c695a8672c8740be1c, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732195508875 2024-11-21T13:25:10,888 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#B#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:10,889 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:10,889 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/644b2b4da29a4d23abbc9efadcdfd961 is 50, key is test_row_0/B:col10/1732195510031/Put/seqid=0 2024-11-21T13:25:10,889 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/159bba570d344bf18675c2e0366c2f6c is 50, key is test_row_0/A:col10/1732195510031/Put/seqid=0 2024-11-21T13:25:10,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-21T13:25:10,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,890 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T13:25:10,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742137_1313 (size=12663) 2024-11-21T13:25:10,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/b7d496252da846a996f74800bea5ce45 is 50, key is test_row_0/A:col10/1732195510067/Put/seqid=0 2024-11-21T13:25:10,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742136_1312 (size=12663) 2024-11-21T13:25:10,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742138_1314 (size=12151) 2024-11-21T13:25:10,921 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/b7d496252da846a996f74800bea5ce45 2024-11-21T13:25:10,923 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/644b2b4da29a4d23abbc9efadcdfd961 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/644b2b4da29a4d23abbc9efadcdfd961 2024-11-21T13:25:10,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/ac637b07b3fe478a8ab39186baa2780d is 50, key is test_row_0/B:col10/1732195510067/Put/seqid=0 2024-11-21T13:25:10,930 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/B of 3fb1b1f418508c88685394456e36ab0b into 644b2b4da29a4d23abbc9efadcdfd961(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:10,930 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:10,930 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/B, priority=13, startTime=1732195510879; duration=0sec 2024-11-21T13:25:10,930 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:10,930 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:B 2024-11-21T13:25:10,930 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:10,932 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:10,932 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/C is initiating minor compaction (all files) 2024-11-21T13:25:10,932 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/C in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:10,932 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/c89dd676f6a746e5b57b6b1ff25c511c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/29ffe480d35c401994863788240479f0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b5dbc203fccf49aca97b7bd576fa048e] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.0 K 2024-11-21T13:25:10,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742139_1315 (size=12151) 2024-11-21T13:25:10,933 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting c89dd676f6a746e5b57b6b1ff25c511c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732195507021 2024-11-21T13:25:10,933 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/ac637b07b3fe478a8ab39186baa2780d 2024-11-21T13:25:10,933 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 29ffe480d35c401994863788240479f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1732195507709 2024-11-21T13:25:10,934 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b5dbc203fccf49aca97b7bd576fa048e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732195508880 2024-11-21T13:25:10,942 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#C#compaction#261 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:10,942 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/85ebe130ff874631909a859c1340f709 is 50, key is test_row_0/C:col10/1732195510031/Put/seqid=0 2024-11-21T13:25:10,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/a9fdf2edcfa141ddbe861922de379723 is 50, key is test_row_0/C:col10/1732195510067/Put/seqid=0 2024-11-21T13:25:10,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742140_1316 (size=12663) 2024-11-21T13:25:10,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742141_1317 (size=12151) 2024-11-21T13:25:11,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T13:25:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:11,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:11,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195571207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,211 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195571209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,213 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195571209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195571209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195571211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,215 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,311 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/159bba570d344bf18675c2e0366c2f6c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/159bba570d344bf18675c2e0366c2f6c 2024-11-21T13:25:11,316 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/A of 3fb1b1f418508c88685394456e36ab0b into 159bba570d344bf18675c2e0366c2f6c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:11,316 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:11,316 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/A, priority=13, startTime=1732195510879; duration=0sec 2024-11-21T13:25:11,316 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:11,316 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:A 2024-11-21T13:25:11,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195571319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,323 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195571319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195571319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,354 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/85ebe130ff874631909a859c1340f709 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/85ebe130ff874631909a859c1340f709 2024-11-21T13:25:11,358 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/C of 3fb1b1f418508c88685394456e36ab0b into 85ebe130ff874631909a859c1340f709(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:11,358 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:11,358 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/C, priority=13, startTime=1732195510880; duration=0sec 2024-11-21T13:25:11,358 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:11,358 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:C 2024-11-21T13:25:11,362 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/a9fdf2edcfa141ddbe861922de379723 2024-11-21T13:25:11,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/b7d496252da846a996f74800bea5ce45 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/b7d496252da846a996f74800bea5ce45 2024-11-21T13:25:11,369 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/b7d496252da846a996f74800bea5ce45, entries=150, sequenceid=230, filesize=11.9 K 2024-11-21T13:25:11,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/ac637b07b3fe478a8ab39186baa2780d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/ac637b07b3fe478a8ab39186baa2780d 2024-11-21T13:25:11,373 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/ac637b07b3fe478a8ab39186baa2780d, entries=150, sequenceid=230, filesize=11.9 K 2024-11-21T13:25:11,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/a9fdf2edcfa141ddbe861922de379723 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/a9fdf2edcfa141ddbe861922de379723 2024-11-21T13:25:11,378 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/a9fdf2edcfa141ddbe861922de379723, entries=150, sequenceid=230, filesize=11.9 K 2024-11-21T13:25:11,379 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 3fb1b1f418508c88685394456e36ab0b in 488ms, sequenceid=230, compaction requested=false 2024-11-21T13:25:11,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:11,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:11,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-21T13:25:11,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-21T13:25:11,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-21T13:25:11,383 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4090 sec 2024-11-21T13:25:11,386 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4150 sec 2024-11-21T13:25:11,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:11,531 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:11,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T13:25:11,537 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/e24e9a8f3fc34fa7b17698291bb6e7a1 is 50, key is test_row_0/A:col10/1732195511530/Put/seqid=0 2024-11-21T13:25:11,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742142_1318 (size=16931) 2024-11-21T13:25:11,541 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/e24e9a8f3fc34fa7b17698291bb6e7a1 2024-11-21T13:25:11,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/4bc93b65c98d4bea9fea53088ec48a4e is 50, key is test_row_0/B:col10/1732195511530/Put/seqid=0 2024-11-21T13:25:11,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742143_1319 (size=12151) 2024-11-21T13:25:11,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/4bc93b65c98d4bea9fea53088ec48a4e 2024-11-21T13:25:11,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/b553425e1408469287ee35daae3fc794 is 50, key is test_row_0/C:col10/1732195511530/Put/seqid=0 2024-11-21T13:25:11,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742144_1320 (size=12151) 2024-11-21T13:25:11,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/b553425e1408469287ee35daae3fc794 2024-11-21T13:25:11,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195571559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195571568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/e24e9a8f3fc34fa7b17698291bb6e7a1 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/e24e9a8f3fc34fa7b17698291bb6e7a1 2024-11-21T13:25:11,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,571 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195571568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/e24e9a8f3fc34fa7b17698291bb6e7a1, entries=250, sequenceid=249, filesize=16.5 K 2024-11-21T13:25:11,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/4bc93b65c98d4bea9fea53088ec48a4e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/4bc93b65c98d4bea9fea53088ec48a4e 2024-11-21T13:25:11,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/4bc93b65c98d4bea9fea53088ec48a4e, entries=150, sequenceid=249, filesize=11.9 K 2024-11-21T13:25:11,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/b553425e1408469287ee35daae3fc794 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b553425e1408469287ee35daae3fc794 2024-11-21T13:25:11,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b553425e1408469287ee35daae3fc794, entries=150, sequenceid=249, filesize=11.9 K 2024-11-21T13:25:11,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3fb1b1f418508c88685394456e36ab0b in 60ms, sequenceid=249, compaction requested=true 2024-11-21T13:25:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:11,591 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:11,591 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:11,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:11,592 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:11,592 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:11,592 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/B is initiating minor compaction (all files) 2024-11-21T13:25:11,592 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/A is initiating minor compaction (all files) 2024-11-21T13:25:11,592 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/B in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:11,592 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/A in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:11,592 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/159bba570d344bf18675c2e0366c2f6c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/b7d496252da846a996f74800bea5ce45, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/e24e9a8f3fc34fa7b17698291bb6e7a1] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=40.8 K 2024-11-21T13:25:11,592 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/644b2b4da29a4d23abbc9efadcdfd961, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/ac637b07b3fe478a8ab39186baa2780d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/4bc93b65c98d4bea9fea53088ec48a4e] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.1 K 2024-11-21T13:25:11,593 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 159bba570d344bf18675c2e0366c2f6c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732195508880 2024-11-21T13:25:11,593 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 644b2b4da29a4d23abbc9efadcdfd961, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732195508880 2024-11-21T13:25:11,593 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7d496252da846a996f74800bea5ce45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732195510064 2024-11-21T13:25:11,593 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting ac637b07b3fe478a8ab39186baa2780d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732195510064 2024-11-21T13:25:11,594 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting e24e9a8f3fc34fa7b17698291bb6e7a1, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732195511204 2024-11-21T13:25:11,594 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bc93b65c98d4bea9fea53088ec48a4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732195511204 2024-11-21T13:25:11,604 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#A#compaction#266 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:11,605 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/24308220dcbb4710aaaed91bb4f31682 is 50, key is test_row_0/A:col10/1732195511530/Put/seqid=0 2024-11-21T13:25:11,608 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:11,609 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/d0a5e6d49ca14c9db224441ca1019be0 is 50, key is test_row_0/B:col10/1732195511530/Put/seqid=0 2024-11-21T13:25:11,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742145_1321 (size=12765) 2024-11-21T13:25:11,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742146_1322 (size=12765) 2024-11-21T13:25:11,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:11,681 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:11,681 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T13:25:11,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/06f59e49a061498a924703921c6525f9 is 50, key is test_row_0/A:col10/1732195511556/Put/seqid=0 2024-11-21T13:25:11,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742147_1323 (size=12301) 2024-11-21T13:25:11,690 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/06f59e49a061498a924703921c6525f9 2024-11-21T13:25:11,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/9e9e948019c04a6fa2f9f1d79d83afae is 50, key is test_row_0/B:col10/1732195511556/Put/seqid=0 2024-11-21T13:25:11,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195571700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195571700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195571702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,706 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742148_1324 (size=12301) 2024-11-21T13:25:11,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/9e9e948019c04a6fa2f9f1d79d83afae 2024-11-21T13:25:11,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/b940453fbd7e4541914ff52fc5ca0eab is 50, key is test_row_0/C:col10/1732195511556/Put/seqid=0 2024-11-21T13:25:11,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742149_1325 (size=12301) 2024-11-21T13:25:11,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195571808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195571809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:11,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195571809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:11,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195572019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195572019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195572019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,024 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/24308220dcbb4710aaaed91bb4f31682 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/24308220dcbb4710aaaed91bb4f31682 2024-11-21T13:25:12,031 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/A of 3fb1b1f418508c88685394456e36ab0b into 24308220dcbb4710aaaed91bb4f31682(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:12,031 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:12,031 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/A, priority=13, startTime=1732195511591; duration=0sec 2024-11-21T13:25:12,031 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:12,031 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:A 2024-11-21T13:25:12,031 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:12,032 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/d0a5e6d49ca14c9db224441ca1019be0 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/d0a5e6d49ca14c9db224441ca1019be0 2024-11-21T13:25:12,033 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:12,033 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/C is initiating minor compaction (all files) 2024-11-21T13:25:12,033 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/C in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:12,033 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/85ebe130ff874631909a859c1340f709, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/a9fdf2edcfa141ddbe861922de379723, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b553425e1408469287ee35daae3fc794] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.1 K 2024-11-21T13:25:12,034 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85ebe130ff874631909a859c1340f709, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732195508880 2024-11-21T13:25:12,034 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9fdf2edcfa141ddbe861922de379723, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732195510064 2024-11-21T13:25:12,035 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting b553425e1408469287ee35daae3fc794, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732195511204 2024-11-21T13:25:12,037 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/B of 3fb1b1f418508c88685394456e36ab0b into d0a5e6d49ca14c9db224441ca1019be0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:12,037 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:12,037 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/B, priority=13, startTime=1732195511591; duration=0sec 2024-11-21T13:25:12,037 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:12,037 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:B 2024-11-21T13:25:12,055 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#C#compaction#271 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:12,055 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/553b8486dced43888fa53c7f576ed249 is 50, key is test_row_0/C:col10/1732195511530/Put/seqid=0 2024-11-21T13:25:12,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742150_1326 (size=12765) 2024-11-21T13:25:12,092 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/553b8486dced43888fa53c7f576ed249 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/553b8486dced43888fa53c7f576ed249 2024-11-21T13:25:12,100 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/C of 3fb1b1f418508c88685394456e36ab0b into 553b8486dced43888fa53c7f576ed249(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:12,100 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:12,100 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/C, priority=13, startTime=1732195511591; duration=0sec 2024-11-21T13:25:12,100 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:12,100 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:C 2024-11-21T13:25:12,125 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/b940453fbd7e4541914ff52fc5ca0eab 2024-11-21T13:25:12,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-21T13:25:12,130 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:12,131 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:12,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/06f59e49a061498a924703921c6525f9 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/06f59e49a061498a924703921c6525f9 2024-11-21T13:25:12,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-21T13:25:12,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T13:25:12,135 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:12,138 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:12,138 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:12,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/06f59e49a061498a924703921c6525f9, entries=150, sequenceid=271, filesize=12.0 K 2024-11-21T13:25:12,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/9e9e948019c04a6fa2f9f1d79d83afae as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9e9e948019c04a6fa2f9f1d79d83afae 2024-11-21T13:25:12,145 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9e9e948019c04a6fa2f9f1d79d83afae, entries=150, sequenceid=271, filesize=12.0 K 2024-11-21T13:25:12,146 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/b940453fbd7e4541914ff52fc5ca0eab as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b940453fbd7e4541914ff52fc5ca0eab 2024-11-21T13:25:12,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b940453fbd7e4541914ff52fc5ca0eab, entries=150, sequenceid=271, filesize=12.0 K 2024-11-21T13:25:12,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 3fb1b1f418508c88685394456e36ab0b in 470ms, sequenceid=271, compaction requested=false 2024-11-21T13:25:12,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:12,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:12,233 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:12,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T13:25:12,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/128f98c347ac436bb267683e43e93edc is 50, key is test_row_0/A:col10/1732195512231/Put/seqid=0 2024-11-21T13:25:12,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T13:25:12,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742151_1327 (size=17181) 2024-11-21T13:25:12,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/128f98c347ac436bb267683e43e93edc 2024-11-21T13:25:12,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/4f77c60a5b8640f4895af3d222276cc1 is 50, key is test_row_0/B:col10/1732195512231/Put/seqid=0 2024-11-21T13:25:12,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195572275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195572278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742152_1328 (size=12301) 2024-11-21T13:25:12,291 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T13:25:12,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:12,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:12,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:12,291 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:12,292 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:12,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:12,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/4f77c60a5b8640f4895af3d222276cc1 2024-11-21T13:25:12,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/c03271f454204da286995014b5060d64 is 50, key is test_row_0/C:col10/1732195512231/Put/seqid=0 2024-11-21T13:25:12,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742153_1329 (size=12301) 2024-11-21T13:25:12,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195572329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195572329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195572329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195572389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195572389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T13:25:12,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:12,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:12,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:12,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:12,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T13:25:12,598 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T13:25:12,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:12,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:12,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:12,598 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:12,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:12,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195572599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195572599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/c03271f454204da286995014b5060d64 2024-11-21T13:25:12,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/128f98c347ac436bb267683e43e93edc as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/128f98c347ac436bb267683e43e93edc 2024-11-21T13:25:12,721 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/128f98c347ac436bb267683e43e93edc, entries=250, sequenceid=290, filesize=16.8 K 2024-11-21T13:25:12,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/4f77c60a5b8640f4895af3d222276cc1 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/4f77c60a5b8640f4895af3d222276cc1 2024-11-21T13:25:12,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/4f77c60a5b8640f4895af3d222276cc1, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T13:25:12,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/c03271f454204da286995014b5060d64 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/c03271f454204da286995014b5060d64 2024-11-21T13:25:12,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/c03271f454204da286995014b5060d64, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T13:25:12,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 3fb1b1f418508c88685394456e36ab0b in 500ms, sequenceid=290, compaction requested=true 2024-11-21T13:25:12,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:12,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:12,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:12,734 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:12,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:12,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:12,734 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:12,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:12,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:12,735 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:12,735 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42247 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:12,735 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/A is initiating minor compaction (all files) 2024-11-21T13:25:12,735 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/B is initiating minor compaction (all files) 2024-11-21T13:25:12,735 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/A in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:12,735 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/24308220dcbb4710aaaed91bb4f31682, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/06f59e49a061498a924703921c6525f9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/128f98c347ac436bb267683e43e93edc] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=41.3 K 2024-11-21T13:25:12,735 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/B in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:12,735 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/d0a5e6d49ca14c9db224441ca1019be0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9e9e948019c04a6fa2f9f1d79d83afae, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/4f77c60a5b8640f4895af3d222276cc1] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.5 K 2024-11-21T13:25:12,735 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24308220dcbb4710aaaed91bb4f31682, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732195511204 2024-11-21T13:25:12,735 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting d0a5e6d49ca14c9db224441ca1019be0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732195511204 2024-11-21T13:25:12,735 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06f59e49a061498a924703921c6525f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732195511556 2024-11-21T13:25:12,735 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e9e948019c04a6fa2f9f1d79d83afae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732195511556 2024-11-21T13:25:12,736 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 128f98c347ac436bb267683e43e93edc, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732195511692 2024-11-21T13:25:12,736 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f77c60a5b8640f4895af3d222276cc1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732195511700 2024-11-21T13:25:12,742 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#A#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:12,742 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/98713d13f929467e80ccb85d120189ef is 50, key is test_row_0/A:col10/1732195512231/Put/seqid=0 2024-11-21T13:25:12,746 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#B#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:12,747 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/8599f8f7c819423d89dc1dc64b7b9c01 is 50, key is test_row_0/B:col10/1732195512231/Put/seqid=0 2024-11-21T13:25:12,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742154_1330 (size=13017) 2024-11-21T13:25:12,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-21T13:25:12,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:12,756 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T13:25:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T13:25:12,760 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/98713d13f929467e80ccb85d120189ef as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/98713d13f929467e80ccb85d120189ef 2024-11-21T13:25:12,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/06e636cdb1cd4f8f8e94c364e374dd27 is 50, key is test_row_0/A:col10/1732195512276/Put/seqid=0 2024-11-21T13:25:12,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742155_1331 (size=13017) 2024-11-21T13:25:12,764 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/A of 3fb1b1f418508c88685394456e36ab0b into 98713d13f929467e80ccb85d120189ef(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:12,765 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:12,765 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/A, priority=13, startTime=1732195512734; duration=0sec 2024-11-21T13:25:12,765 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:12,765 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:A 2024-11-21T13:25:12,765 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:12,766 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:12,766 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/C is initiating minor compaction (all files) 2024-11-21T13:25:12,766 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/C in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:12,766 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/553b8486dced43888fa53c7f576ed249, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b940453fbd7e4541914ff52fc5ca0eab, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/c03271f454204da286995014b5060d64] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.5 K 2024-11-21T13:25:12,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742156_1332 (size=12301) 2024-11-21T13:25:12,766 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 553b8486dced43888fa53c7f576ed249, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1732195511204 2024-11-21T13:25:12,767 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting b940453fbd7e4541914ff52fc5ca0eab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1732195511556 2024-11-21T13:25:12,767 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting c03271f454204da286995014b5060d64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732195511700 2024-11-21T13:25:12,767 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/06e636cdb1cd4f8f8e94c364e374dd27 2024-11-21T13:25:12,774 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#C#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:12,775 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/26f48451fe05441d9e3575777da6e55b is 50, key is test_row_0/C:col10/1732195512231/Put/seqid=0 2024-11-21T13:25:12,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/16f35a6844c2416f939b540c07b13cd1 is 50, key is test_row_0/B:col10/1732195512276/Put/seqid=0 2024-11-21T13:25:12,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742157_1333 (size=13017) 2024-11-21T13:25:12,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742158_1334 (size=12301) 2024-11-21T13:25:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:12,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:12,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195572857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195572857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,862 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195572861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195572909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,912 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195572909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,912 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195572969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195572969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:12,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:12,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195572969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:12,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:12,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:13,165 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/8599f8f7c819423d89dc1dc64b7b9c01 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/8599f8f7c819423d89dc1dc64b7b9c01 2024-11-21T13:25:13,170 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/B of 3fb1b1f418508c88685394456e36ab0b into 8599f8f7c819423d89dc1dc64b7b9c01(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:13,170 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:13,170 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/B, priority=13, startTime=1732195512734; duration=0sec 2024-11-21T13:25:13,170 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:13,170 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:B 2024-11-21T13:25:13,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195573179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:13,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195573179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,181 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:13,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:13,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195573179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:13,191 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/26f48451fe05441d9e3575777da6e55b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/26f48451fe05441d9e3575777da6e55b 2024-11-21T13:25:13,194 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/16f35a6844c2416f939b540c07b13cd1 2024-11-21T13:25:13,197 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/C of 3fb1b1f418508c88685394456e36ab0b into 26f48451fe05441d9e3575777da6e55b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:13,197 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:13,197 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/C, priority=13, startTime=1732195512734; duration=0sec 2024-11-21T13:25:13,197 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:13,197 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:C 2024-11-21T13:25:13,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/220fdcfa2624431db02647820274e870 is 50, key is test_row_0/C:col10/1732195512276/Put/seqid=0 2024-11-21T13:25:13,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742159_1335 (size=12301) 2024-11-21T13:25:13,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T13:25:13,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195573419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:13,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195573419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:13,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,421 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:13,421 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,422 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:13,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:13,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195573489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:13,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195573489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:13,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:13,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195573489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:13,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:13,607 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/220fdcfa2624431db02647820274e870 2024-11-21T13:25:13,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/06e636cdb1cd4f8f8e94c364e374dd27 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/06e636cdb1cd4f8f8e94c364e374dd27 2024-11-21T13:25:13,614 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/06e636cdb1cd4f8f8e94c364e374dd27, entries=150, sequenceid=310, filesize=12.0 K 2024-11-21T13:25:13,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/16f35a6844c2416f939b540c07b13cd1 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/16f35a6844c2416f939b540c07b13cd1 2024-11-21T13:25:13,617 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/16f35a6844c2416f939b540c07b13cd1, entries=150, sequenceid=310, filesize=12.0 K 2024-11-21T13:25:13,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/220fdcfa2624431db02647820274e870 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/220fdcfa2624431db02647820274e870 2024-11-21T13:25:13,621 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/220fdcfa2624431db02647820274e870, entries=150, sequenceid=310, filesize=12.0 K 2024-11-21T13:25:13,622 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 3fb1b1f418508c88685394456e36ab0b in 866ms, sequenceid=310, compaction requested=false 2024-11-21T13:25:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:13,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-21T13:25:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-21T13:25:13,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-21T13:25:13,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4850 sec 2024-11-21T13:25:13,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.4940 sec 2024-11-21T13:25:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:14,001 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:14,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T13:25:14,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/427dbcb6b95949c0aeac843a90acdaff is 50, key is test_row_0/A:col10/1732195514000/Put/seqid=0 2024-11-21T13:25:14,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742160_1336 (size=12301) 2024-11-21T13:25:14,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195574049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195574050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195574052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195574158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195574159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195574159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,163 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-21T13:25:14,279 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:14,279 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:14,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-21T13:25:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T13:25:14,282 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:14,283 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:14,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:14,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195574370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195574370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,373 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195574370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T13:25:14,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/427dbcb6b95949c0aeac843a90acdaff 2024-11-21T13:25:14,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/c00f657f5ac846b9b313b1ad1a37acc4 is 50, key is test_row_0/B:col10/1732195514000/Put/seqid=0 2024-11-21T13:25:14,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195574428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742161_1337 (size=12301) 2024-11-21T13:25:14,433 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 3fb1b1f418508c88685394456e36ab0b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:25:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,433 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T13:25:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:14,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:14,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:14,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195574439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,440 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_1' in 3fb1b1f418508c88685394456e36ab0b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,441 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T13:25:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:14,589 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:14,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:14,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T13:25:14,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195574679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195574679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195574679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:14,683 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:14,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T13:25:14,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:14,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:14,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:14,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:14,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:14,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:14,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/c00f657f5ac846b9b313b1ad1a37acc4 2024-11-21T13:25:14,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/9b3f06f05e734b8ea363830aed553df2 is 50, key is test_row_0/C:col10/1732195514000/Put/seqid=0 2024-11-21T13:25:14,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742162_1338 (size=12301) 2024-11-21T13:25:14,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T13:25:14,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:14,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:14,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T13:25:15,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T13:25:15,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:15,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:15,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:15,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:15,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:15,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195575189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:15,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195575189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:15,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:15,191 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:15,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195575189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:15,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:15,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T13:25:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:15,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:15,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:15,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:15,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=332 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/9b3f06f05e734b8ea363830aed553df2 2024-11-21T13:25:15,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/427dbcb6b95949c0aeac843a90acdaff as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/427dbcb6b95949c0aeac843a90acdaff 2024-11-21T13:25:15,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/427dbcb6b95949c0aeac843a90acdaff, entries=150, sequenceid=332, filesize=12.0 K 2024-11-21T13:25:15,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/c00f657f5ac846b9b313b1ad1a37acc4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/c00f657f5ac846b9b313b1ad1a37acc4 2024-11-21T13:25:15,262 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/c00f657f5ac846b9b313b1ad1a37acc4, entries=150, sequenceid=332, filesize=12.0 K 2024-11-21T13:25:15,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/9b3f06f05e734b8ea363830aed553df2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/9b3f06f05e734b8ea363830aed553df2 2024-11-21T13:25:15,266 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/9b3f06f05e734b8ea363830aed553df2, entries=150, sequenceid=332, filesize=12.0 K 2024-11-21T13:25:15,267 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 3fb1b1f418508c88685394456e36ab0b in 1266ms, sequenceid=332, compaction requested=true 2024-11-21T13:25:15,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:15,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:15,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:15,267 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:15,267 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:15,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:15,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:15,267 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:15,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:15,268 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:15,268 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/B is initiating minor compaction (all files) 2024-11-21T13:25:15,268 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/B in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:15,268 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/8599f8f7c819423d89dc1dc64b7b9c01, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/16f35a6844c2416f939b540c07b13cd1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/c00f657f5ac846b9b313b1ad1a37acc4] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.7 K 2024-11-21T13:25:15,269 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:15,269 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/A is initiating minor compaction (all files) 2024-11-21T13:25:15,269 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/A in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:15,269 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 8599f8f7c819423d89dc1dc64b7b9c01, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732195511700 2024-11-21T13:25:15,269 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/98713d13f929467e80ccb85d120189ef, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/06e636cdb1cd4f8f8e94c364e374dd27, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/427dbcb6b95949c0aeac843a90acdaff] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.7 K 2024-11-21T13:25:15,269 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 16f35a6844c2416f939b540c07b13cd1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732195512256 2024-11-21T13:25:15,269 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98713d13f929467e80ccb85d120189ef, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732195511700 2024-11-21T13:25:15,269 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting c00f657f5ac846b9b313b1ad1a37acc4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732195512860 2024-11-21T13:25:15,270 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06e636cdb1cd4f8f8e94c364e374dd27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732195512256 2024-11-21T13:25:15,270 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 427dbcb6b95949c0aeac843a90acdaff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732195512860 2024-11-21T13:25:15,286 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#B#compaction#284 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:15,287 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/651a55696af148d58543ee48f6ef3f5c is 50, key is test_row_0/B:col10/1732195514000/Put/seqid=0 2024-11-21T13:25:15,290 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#A#compaction#285 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:15,290 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/7055744dae6943fca9064ffe3d4c6081 is 50, key is test_row_0/A:col10/1732195514000/Put/seqid=0 2024-11-21T13:25:15,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742163_1339 (size=13119) 2024-11-21T13:25:15,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742164_1340 (size=13119) 2024-11-21T13:25:15,308 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/651a55696af148d58543ee48f6ef3f5c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/651a55696af148d58543ee48f6ef3f5c 2024-11-21T13:25:15,313 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/B of 3fb1b1f418508c88685394456e36ab0b into 651a55696af148d58543ee48f6ef3f5c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:15,313 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:15,313 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/B, priority=13, startTime=1732195515267; duration=0sec 2024-11-21T13:25:15,313 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:15,313 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:B 2024-11-21T13:25:15,313 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:15,314 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:15,314 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/C is initiating minor compaction (all files) 2024-11-21T13:25:15,314 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/C in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:15,314 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/26f48451fe05441d9e3575777da6e55b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/220fdcfa2624431db02647820274e870, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/9b3f06f05e734b8ea363830aed553df2] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.7 K 2024-11-21T13:25:15,315 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 26f48451fe05441d9e3575777da6e55b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732195511700 2024-11-21T13:25:15,315 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 220fdcfa2624431db02647820274e870, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1732195512256 2024-11-21T13:25:15,315 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b3f06f05e734b8ea363830aed553df2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732195512860 2024-11-21T13:25:15,321 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#C#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:15,322 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/8d1b015ee30047d2afb3528786acc04b is 50, key is test_row_0/C:col10/1732195514000/Put/seqid=0 2024-11-21T13:25:15,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742165_1341 (size=13119) 2024-11-21T13:25:15,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-21T13:25:15,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:15,353 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T13:25:15,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/cf1aa2e35e80448996d3ce22b90de903 is 50, key is test_row_0/A:col10/1732195514044/Put/seqid=0 2024-11-21T13:25:15,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742166_1342 (size=12301) 2024-11-21T13:25:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T13:25:15,702 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/7055744dae6943fca9064ffe3d4c6081 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/7055744dae6943fca9064ffe3d4c6081 2024-11-21T13:25:15,705 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/A of 3fb1b1f418508c88685394456e36ab0b into 7055744dae6943fca9064ffe3d4c6081(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:15,705 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:15,705 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/A, priority=13, startTime=1732195515267; duration=0sec 2024-11-21T13:25:15,705 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:15,705 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:A 2024-11-21T13:25:15,729 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/8d1b015ee30047d2afb3528786acc04b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/8d1b015ee30047d2afb3528786acc04b 2024-11-21T13:25:15,740 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/C of 3fb1b1f418508c88685394456e36ab0b into 8d1b015ee30047d2afb3528786acc04b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:15,740 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:15,740 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/C, priority=13, startTime=1732195515267; duration=0sec 2024-11-21T13:25:15,740 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:15,740 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:C 2024-11-21T13:25:15,766 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/cf1aa2e35e80448996d3ce22b90de903 2024-11-21T13:25:15,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/594722d1898340168de487c6b405b892 is 50, key is test_row_0/B:col10/1732195514044/Put/seqid=0 2024-11-21T13:25:15,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742167_1343 (size=12301) 2024-11-21T13:25:15,777 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/594722d1898340168de487c6b405b892 2024-11-21T13:25:15,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/d8ce758d10e7454c86379ee309a2b5e7 is 50, key is test_row_0/C:col10/1732195514044/Put/seqid=0 2024-11-21T13:25:15,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742168_1344 (size=12301) 2024-11-21T13:25:15,800 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/d8ce758d10e7454c86379ee309a2b5e7 2024-11-21T13:25:15,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/cf1aa2e35e80448996d3ce22b90de903 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/cf1aa2e35e80448996d3ce22b90de903 2024-11-21T13:25:15,808 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/cf1aa2e35e80448996d3ce22b90de903, entries=150, sequenceid=350, filesize=12.0 K 2024-11-21T13:25:15,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/594722d1898340168de487c6b405b892 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/594722d1898340168de487c6b405b892 2024-11-21T13:25:15,814 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/594722d1898340168de487c6b405b892, entries=150, sequenceid=350, filesize=12.0 K 2024-11-21T13:25:15,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/d8ce758d10e7454c86379ee309a2b5e7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/d8ce758d10e7454c86379ee309a2b5e7 2024-11-21T13:25:15,824 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/d8ce758d10e7454c86379ee309a2b5e7, entries=150, sequenceid=350, filesize=12.0 K 2024-11-21T13:25:15,825 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for 3fb1b1f418508c88685394456e36ab0b in 472ms, sequenceid=350, compaction requested=false 2024-11-21T13:25:15,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:15,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:15,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-21T13:25:15,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-21T13:25:15,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-21T13:25:15,831 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5450 sec 2024-11-21T13:25:15,833 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.5530 sec 2024-11-21T13:25:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:16,210 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:16,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:25:16,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/33c16d701fb1413f8de554a3f7837c63 is 50, key is test_row_0/A:col10/1732195516209/Put/seqid=0 2024-11-21T13:25:16,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742169_1345 (size=14741) 2024-11-21T13:25:16,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195576239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195576240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195576240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195576348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195576349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195576349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-21T13:25:16,430 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:16,430 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:16,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-21T13:25:16,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T13:25:16,432 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:16,432 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:16,432 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:16,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46726 deadline: 1732195576439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,441 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 3fb1b1f418508c88685394456e36ab0b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:25:16,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46758 deadline: 1732195576459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,461 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_1' in 3fb1b1f418508c88685394456e36ab0b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:25:16,461 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,462 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T13:25:16,559 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195576559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195576559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195576559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T13:25:16,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:16,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:16,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:16,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:16,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:16,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/33c16d701fb1413f8de554a3f7837c63 2024-11-21T13:25:16,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/e203d6ac615444b085c9badb504956aa is 50, key is test_row_0/B:col10/1732195516209/Put/seqid=0 2024-11-21T13:25:16,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742170_1346 (size=12301) 2024-11-21T13:25:16,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/e203d6ac615444b085c9badb504956aa 2024-11-21T13:25:16,638 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/65eaff3bfe1141e48491c45d3066e933 is 50, key is test_row_0/C:col10/1732195516209/Put/seqid=0 2024-11-21T13:25:16,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742171_1347 (size=12301) 2024-11-21T13:25:16,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T13:25:16,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:16,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:16,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:16,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:16,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:16,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:16,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T13:25:16,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195576869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195576869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:16,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195576870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:16,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:16,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:16,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T13:25:16,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:16,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:16,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:16,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:16,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:17,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/65eaff3bfe1141e48491c45d3066e933 2024-11-21T13:25:17,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T13:25:17,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:17,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:17,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:17,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:17,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:17,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/33c16d701fb1413f8de554a3f7837c63 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/33c16d701fb1413f8de554a3f7837c63 2024-11-21T13:25:17,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/33c16d701fb1413f8de554a3f7837c63, entries=200, sequenceid=363, filesize=14.4 K 2024-11-21T13:25:17,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/e203d6ac615444b085c9badb504956aa as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/e203d6ac615444b085c9badb504956aa 2024-11-21T13:25:17,054 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/e203d6ac615444b085c9badb504956aa, entries=150, sequenceid=363, filesize=12.0 K 2024-11-21T13:25:17,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/65eaff3bfe1141e48491c45d3066e933 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/65eaff3bfe1141e48491c45d3066e933 2024-11-21T13:25:17,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/65eaff3bfe1141e48491c45d3066e933, entries=150, sequenceid=363, filesize=12.0 K 2024-11-21T13:25:17,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 3fb1b1f418508c88685394456e36ab0b in 848ms, sequenceid=363, compaction requested=true 2024-11-21T13:25:17,059 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:17,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:17,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:17,059 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:17,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:17,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:17,059 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:17,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:17,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:17,060 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:17,060 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:17,060 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/A is initiating minor compaction (all files) 2024-11-21T13:25:17,060 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/B is initiating minor compaction (all files) 2024-11-21T13:25:17,060 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/B in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:17,060 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/A in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:17,060 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/7055744dae6943fca9064ffe3d4c6081, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/cf1aa2e35e80448996d3ce22b90de903, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/33c16d701fb1413f8de554a3f7837c63] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=39.2 K 2024-11-21T13:25:17,060 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/651a55696af148d58543ee48f6ef3f5c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/594722d1898340168de487c6b405b892, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/e203d6ac615444b085c9badb504956aa] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.8 K 2024-11-21T13:25:17,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T13:25:17,060 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 651a55696af148d58543ee48f6ef3f5c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732195512860 2024-11-21T13:25:17,060 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7055744dae6943fca9064ffe3d4c6081, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732195512860 2024-11-21T13:25:17,061 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 594722d1898340168de487c6b405b892, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732195514044 2024-11-21T13:25:17,061 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf1aa2e35e80448996d3ce22b90de903, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732195514044 2024-11-21T13:25:17,061 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e203d6ac615444b085c9badb504956aa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732195516202 2024-11-21T13:25:17,061 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33c16d701fb1413f8de554a3f7837c63, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732195516202 2024-11-21T13:25:17,067 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#A#compaction#293 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:17,067 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#B#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:17,067 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/35cded11313b4ffa9fad260361eea300 is 50, key is test_row_0/B:col10/1732195516209/Put/seqid=0 2024-11-21T13:25:17,067 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/84be355a1c494e3480bdfe0a3efddb7c is 50, key is test_row_0/A:col10/1732195516209/Put/seqid=0 2024-11-21T13:25:17,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742173_1349 (size=13221) 2024-11-21T13:25:17,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742172_1348 (size=13221) 2024-11-21T13:25:17,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-21T13:25:17,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:17,200 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T13:25:17,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/13e998b2532f45dfaf301217385a14b7 is 50, key is test_row_0/A:col10/1732195516238/Put/seqid=0 2024-11-21T13:25:17,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742174_1350 (size=12301) 2024-11-21T13:25:17,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:17,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:17,396 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:17,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195577392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,397 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:17,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:17,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195577394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:17,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:17,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195577394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,399 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:17,488 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/35cded11313b4ffa9fad260361eea300 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/35cded11313b4ffa9fad260361eea300 2024-11-21T13:25:17,491 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/84be355a1c494e3480bdfe0a3efddb7c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/84be355a1c494e3480bdfe0a3efddb7c 2024-11-21T13:25:17,491 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/B of 3fb1b1f418508c88685394456e36ab0b into 35cded11313b4ffa9fad260361eea300(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:17,491 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:17,491 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/B, priority=13, startTime=1732195517059; duration=0sec 2024-11-21T13:25:17,491 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:17,491 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:B 2024-11-21T13:25:17,491 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:17,492 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:17,492 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/C is initiating minor compaction (all files) 2024-11-21T13:25:17,492 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/C in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:17,492 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/8d1b015ee30047d2afb3528786acc04b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/d8ce758d10e7454c86379ee309a2b5e7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/65eaff3bfe1141e48491c45d3066e933] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.8 K 2024-11-21T13:25:17,493 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d1b015ee30047d2afb3528786acc04b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=332, earliestPutTs=1732195512860 2024-11-21T13:25:17,493 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting d8ce758d10e7454c86379ee309a2b5e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=350, earliestPutTs=1732195514044 2024-11-21T13:25:17,493 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 65eaff3bfe1141e48491c45d3066e933, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732195516202 2024-11-21T13:25:17,497 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/A of 3fb1b1f418508c88685394456e36ab0b into 84be355a1c494e3480bdfe0a3efddb7c(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:17,497 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:17,497 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/A, priority=13, startTime=1732195517059; duration=0sec 2024-11-21T13:25:17,497 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:17,497 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:A 2024-11-21T13:25:17,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:17,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195577499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:17,502 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#C#compaction#296 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:17,502 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/8142e9b644354e7fa41c7271f6692d75 is 50, key is test_row_0/C:col10/1732195516209/Put/seqid=0 2024-11-21T13:25:17,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742175_1351 (size=13221) 2024-11-21T13:25:17,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195577509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:17,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195577509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:17,511 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:17,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T13:25:17,609 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/13e998b2532f45dfaf301217385a14b7 2024-11-21T13:25:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/22ab53e7fa0c4891aad1f37177d62df8 is 50, key is test_row_0/B:col10/1732195516238/Put/seqid=0 2024-11-21T13:25:17,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742176_1352 (size=12301) 2024-11-21T13:25:17,622 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/22ab53e7fa0c4891aad1f37177d62df8 2024-11-21T13:25:17,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/001ded2db28849828ca2ed2181900ead is 50, key is test_row_0/C:col10/1732195516238/Put/seqid=0 2024-11-21T13:25:17,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742177_1353 (size=12301) 2024-11-21T13:25:17,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195577709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:17,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195577719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:17,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195577719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:17,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:17,911 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/8142e9b644354e7fa41c7271f6692d75 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/8142e9b644354e7fa41c7271f6692d75 2024-11-21T13:25:17,914 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/C of 3fb1b1f418508c88685394456e36ab0b into 8142e9b644354e7fa41c7271f6692d75(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:17,914 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:17,915 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/C, priority=13, startTime=1732195517059; duration=0sec 2024-11-21T13:25:17,915 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:17,915 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:C 2024-11-21T13:25:18,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:18,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195578019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:18,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195578030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:18,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195578030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:18,032 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/001ded2db28849828ca2ed2181900ead 2024-11-21T13:25:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:18,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/13e998b2532f45dfaf301217385a14b7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/13e998b2532f45dfaf301217385a14b7 2024-11-21T13:25:18,040 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/13e998b2532f45dfaf301217385a14b7, entries=150, sequenceid=388, filesize=12.0 K 2024-11-21T13:25:18,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/22ab53e7fa0c4891aad1f37177d62df8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/22ab53e7fa0c4891aad1f37177d62df8 2024-11-21T13:25:18,044 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/22ab53e7fa0c4891aad1f37177d62df8, entries=150, sequenceid=388, filesize=12.0 K 2024-11-21T13:25:18,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/001ded2db28849828ca2ed2181900ead as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/001ded2db28849828ca2ed2181900ead 2024-11-21T13:25:18,049 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/001ded2db28849828ca2ed2181900ead, entries=150, sequenceid=388, filesize=12.0 K 2024-11-21T13:25:18,049 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 3fb1b1f418508c88685394456e36ab0b in 849ms, sequenceid=388, compaction requested=false 2024-11-21T13:25:18,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:18,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:18,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-21T13:25:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-21T13:25:18,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-21T13:25:18,052 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6180 sec 2024-11-21T13:25:18,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.6230 sec 2024-11-21T13:25:18,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:18,530 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:18,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T13:25:18,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/430a97e2fc20483fad15947d1a41c12b is 50, key is test_row_0/A:col10/1732195517391/Put/seqid=0 2024-11-21T13:25:18,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742178_1354 (size=12301) 2024-11-21T13:25:18,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195578563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:18,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195578564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:18,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:18,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195578564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:18,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-21T13:25:18,580 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:18,580 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-21T13:25:18,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T13:25:18,581 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:18,582 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:18,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:18,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195578669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:18,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:18,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195578669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:18,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195578669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:18,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T13:25:18,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T13:25:18,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:18,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:18,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:18,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:18,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:18,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:18,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195578879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:18,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:18,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195578879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:18,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:18,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195578879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:18,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:18,887 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T13:25:18,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:18,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:18,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:18,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:18,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:18,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T13:25:18,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/430a97e2fc20483fad15947d1a41c12b 2024-11-21T13:25:18,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/6eb98971950a4f2e991a934141c29c6c is 50, key is test_row_0/B:col10/1732195517391/Put/seqid=0 2024-11-21T13:25:18,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742179_1355 (size=12301) 2024-11-21T13:25:19,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T13:25:19,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:19,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:19,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:19,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:19,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:19,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195579189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:19,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T13:25:19,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:19,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195579189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:19,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:19,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:19,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195579189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:19,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:19,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:19,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:19,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:19,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T13:25:19,230 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T13:25:19,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T13:25:19,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:19,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:19,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:19,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:19,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:19,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/6eb98971950a4f2e991a934141c29c6c 2024-11-21T13:25:19,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/34b25a1204dc417c9d39ae18d1e8e36f is 50, key is test_row_0/C:col10/1732195517391/Put/seqid=0 2024-11-21T13:25:19,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742180_1356 (size=12301) 2024-11-21T13:25:19,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/34b25a1204dc417c9d39ae18d1e8e36f 2024-11-21T13:25:19,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/430a97e2fc20483fad15947d1a41c12b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/430a97e2fc20483fad15947d1a41c12b 2024-11-21T13:25:19,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/430a97e2fc20483fad15947d1a41c12b, entries=150, sequenceid=404, filesize=12.0 K 2024-11-21T13:25:19,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/6eb98971950a4f2e991a934141c29c6c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/6eb98971950a4f2e991a934141c29c6c 2024-11-21T13:25:19,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/6eb98971950a4f2e991a934141c29c6c, entries=150, sequenceid=404, filesize=12.0 K 2024-11-21T13:25:19,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/34b25a1204dc417c9d39ae18d1e8e36f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/34b25a1204dc417c9d39ae18d1e8e36f 2024-11-21T13:25:19,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/34b25a1204dc417c9d39ae18d1e8e36f, entries=150, sequenceid=404, filesize=12.0 K 2024-11-21T13:25:19,398 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 3fb1b1f418508c88685394456e36ab0b in 868ms, sequenceid=404, compaction requested=true 2024-11-21T13:25:19,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:19,399 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:19,399 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:19,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:19,402 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:19,402 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/B is initiating minor compaction (all files) 2024-11-21T13:25:19,402 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/B in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:19,402 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/35cded11313b4ffa9fad260361eea300, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/22ab53e7fa0c4891aad1f37177d62df8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/6eb98971950a4f2e991a934141c29c6c] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.9 K 2024-11-21T13:25:19,402 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 35cded11313b4ffa9fad260361eea300, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732195516202 2024-11-21T13:25:19,402 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:19,403 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/A is initiating minor compaction (all files) 2024-11-21T13:25:19,403 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/A in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:19,403 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/84be355a1c494e3480bdfe0a3efddb7c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/13e998b2532f45dfaf301217385a14b7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/430a97e2fc20483fad15947d1a41c12b] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.9 K 2024-11-21T13:25:19,403 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 22ab53e7fa0c4891aad1f37177d62df8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732195516238 2024-11-21T13:25:19,404 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb98971950a4f2e991a934141c29c6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732195517391 2024-11-21T13:25:19,404 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84be355a1c494e3480bdfe0a3efddb7c, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732195516202 2024-11-21T13:25:19,404 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13e998b2532f45dfaf301217385a14b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732195516238 2024-11-21T13:25:19,406 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 430a97e2fc20483fad15947d1a41c12b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732195517391 2024-11-21T13:25:19,413 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#B#compaction#302 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:19,413 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/35a9333c60be46c2afc7fbc1a39bd99d is 50, key is test_row_0/B:col10/1732195517391/Put/seqid=0 2024-11-21T13:25:19,415 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#A#compaction#303 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:19,415 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/6df564f859684feb9ac5a7241852f6e5 is 50, key is test_row_0/A:col10/1732195517391/Put/seqid=0 2024-11-21T13:25:19,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742181_1357 (size=13323) 2024-11-21T13:25:19,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742182_1358 (size=13323) 2024-11-21T13:25:19,448 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/6df564f859684feb9ac5a7241852f6e5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/6df564f859684feb9ac5a7241852f6e5 2024-11-21T13:25:19,452 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/A of 3fb1b1f418508c88685394456e36ab0b into 6df564f859684feb9ac5a7241852f6e5(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:19,452 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:19,452 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/A, priority=13, startTime=1732195519399; duration=0sec 2024-11-21T13:25:19,452 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:19,452 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:A 2024-11-21T13:25:19,452 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:19,453 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37823 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:19,453 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 3fb1b1f418508c88685394456e36ab0b/C is initiating minor compaction (all files) 2024-11-21T13:25:19,453 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 3fb1b1f418508c88685394456e36ab0b/C in TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:19,453 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/8142e9b644354e7fa41c7271f6692d75, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/001ded2db28849828ca2ed2181900ead, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/34b25a1204dc417c9d39ae18d1e8e36f] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp, totalSize=36.9 K 2024-11-21T13:25:19,453 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8142e9b644354e7fa41c7271f6692d75, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1732195516202 2024-11-21T13:25:19,454 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 001ded2db28849828ca2ed2181900ead, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732195516238 2024-11-21T13:25:19,454 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34b25a1204dc417c9d39ae18d1e8e36f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1732195517391 2024-11-21T13:25:19,461 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 3fb1b1f418508c88685394456e36ab0b#C#compaction#304 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:19,461 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/b4a9a49f860946a1beb65812cd7e6d39 is 50, key is test_row_0/C:col10/1732195517391/Put/seqid=0 2024-11-21T13:25:19,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742183_1359 (size=13323) 2024-11-21T13:25:19,474 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/b4a9a49f860946a1beb65812cd7e6d39 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b4a9a49f860946a1beb65812cd7e6d39 2024-11-21T13:25:19,480 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/C of 3fb1b1f418508c88685394456e36ab0b into b4a9a49f860946a1beb65812cd7e6d39(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:19,480 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:19,480 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/C, priority=13, startTime=1732195519399; duration=0sec 2024-11-21T13:25:19,480 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:19,480 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:C 2024-11-21T13:25:19,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-21T13:25:19,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:19,503 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T13:25:19,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/b5c0bc69fddc4698a29dbf6579b4a55e is 50, key is test_row_0/A:col10/1732195518563/Put/seqid=0 2024-11-21T13:25:19,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742184_1360 (size=12301) 2024-11-21T13:25:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:19,701 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. as already flushing 2024-11-21T13:25:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T13:25:19,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195579740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:19,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195579740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,743 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:19,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195579740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:19,835 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/35a9333c60be46c2afc7fbc1a39bd99d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/35a9333c60be46c2afc7fbc1a39bd99d 2024-11-21T13:25:19,841 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 3fb1b1f418508c88685394456e36ab0b/B of 3fb1b1f418508c88685394456e36ab0b into 35a9333c60be46c2afc7fbc1a39bd99d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:19,841 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:19,841 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., storeName=3fb1b1f418508c88685394456e36ab0b/B, priority=13, startTime=1732195519399; duration=0sec 2024-11-21T13:25:19,841 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:19,841 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:B 2024-11-21T13:25:19,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195579849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:19,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:19,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195579849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:19,851 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:19,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195579849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:19,852 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:19,910 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/b5c0bc69fddc4698a29dbf6579b4a55e 2024-11-21T13:25:19,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/f1a633c699db47d9862d6bd3591bbfcd is 50, key is test_row_0/B:col10/1732195518563/Put/seqid=0 2024-11-21T13:25:19,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742185_1361 (size=12301) 2024-11-21T13:25:19,922 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/f1a633c699db47d9862d6bd3591bbfcd 2024-11-21T13:25:19,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/0b9351315a7f41e981546fc5b5a92ea4 is 50, key is test_row_0/C:col10/1732195518563/Put/seqid=0 2024-11-21T13:25:19,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742186_1362 (size=12301) 2024-11-21T13:25:20,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46790 deadline: 1732195580059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:20,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46774 deadline: 1732195580059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:20,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:20,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:46720 deadline: 1732195580059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:20,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:20,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:20,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:20,063 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:20,254 INFO [Thread-1236 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1236. 2024-11-21T13:25:20,255 DEBUG [Thread-1236 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:20,255 DEBUG [Thread-1236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,255 DEBUG [Thread-1236 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:20,255 INFO [Thread-1240 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1240. 2024-11-21T13:25:20,255 DEBUG [Thread-1240 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:20,255 DEBUG [Thread-1240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:20,255 DEBUG [Thread-1240 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,256 INFO [Thread-1242 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1242. 2024-11-21T13:25:20,256 DEBUG [Thread-1242 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:20,256 DEBUG [Thread-1242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,256 DEBUG [Thread-1242 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,256 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:20,257 INFO [Thread-1238 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1238. 2024-11-21T13:25:20,257 DEBUG [Thread-1238 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:20,257 DEBUG [Thread-1238 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,257 DEBUG [Thread-1238 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:20,257 INFO [Thread-1234 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1234. 2024-11-21T13:25:20,257 DEBUG [Thread-1234 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:20,257 DEBUG [Thread-1234 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,258 DEBUG [Thread-1234 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,258 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:20,334 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/0b9351315a7f41e981546fc5b5a92ea4 2024-11-21T13:25:20,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/b5c0bc69fddc4698a29dbf6579b4a55e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/b5c0bc69fddc4698a29dbf6579b4a55e 2024-11-21T13:25:20,347 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/b5c0bc69fddc4698a29dbf6579b4a55e, entries=150, sequenceid=429, filesize=12.0 K 2024-11-21T13:25:20,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/f1a633c699db47d9862d6bd3591bbfcd as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/f1a633c699db47d9862d6bd3591bbfcd 2024-11-21T13:25:20,351 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/f1a633c699db47d9862d6bd3591bbfcd, entries=150, sequenceid=429, filesize=12.0 K 2024-11-21T13:25:20,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/0b9351315a7f41e981546fc5b5a92ea4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/0b9351315a7f41e981546fc5b5a92ea4 2024-11-21T13:25:20,354 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/0b9351315a7f41e981546fc5b5a92ea4, entries=150, sequenceid=429, filesize=12.0 K 2024-11-21T13:25:20,355 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for 3fb1b1f418508c88685394456e36ab0b in 852ms, sequenceid=429, compaction requested=false 2024-11-21T13:25:20,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:20,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:20,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-21T13:25:20,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-21T13:25:20,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-21T13:25:20,358 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7740 sec 2024-11-21T13:25:20,359 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.7780 sec 2024-11-21T13:25:20,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:20,370 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:20,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-21T13:25:20,371 INFO [Thread-1227 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1227. 2024-11-21T13:25:20,371 INFO [Thread-1231 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1231. 2024-11-21T13:25:20,371 DEBUG [Thread-1227 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:20,371 DEBUG [Thread-1231 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:20,371 DEBUG [Thread-1227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,371 DEBUG [Thread-1231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,371 DEBUG [Thread-1231 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,371 DEBUG [Thread-1227 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,371 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:20,371 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:20,372 INFO [Thread-1229 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1229. 2024-11-21T13:25:20,372 DEBUG [Thread-1229 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:20,372 DEBUG [Thread-1229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,372 DEBUG [Thread-1229 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:20,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/92e6f855b8674fd5b9bfc9509acb5a68 is 50, key is test_row_0/A:col10/1732195520369/Put/seqid=0 2024-11-21T13:25:20,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742187_1363 (size=12301) 2024-11-21T13:25:20,481 INFO [Thread-1223 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1223. 2024-11-21T13:25:20,481 DEBUG [Thread-1223 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:20,481 DEBUG [Thread-1223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,482 DEBUG [Thread-1223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:20,492 INFO [Thread-1225 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1225. 2024-11-21T13:25:20,492 DEBUG [Thread-1225 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:20,492 DEBUG [Thread-1225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,492 DEBUG [Thread-1225 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,492 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:20,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-21T13:25:20,741 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:20,741 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T13:25:20,741 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 65 2024-11-21T13:25:20,741 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-21T13:25:20,741 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-21T13:25:20,741 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 82 2024-11-21T13:25:20,741 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-21T13:25:20,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T13:25:20,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4223 2024-11-21T13:25:20,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4182 2024-11-21T13:25:20,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4176 2024-11-21T13:25:20,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4216 2024-11-21T13:25:20,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4197 2024-11-21T13:25:20,742 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T13:25:20,742 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T13:25:20,742 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T13:25:20,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:20,743 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:20,745 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T13:25:20,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T13:25:20,752 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195520752"}]},"ts":"1732195520752"} 2024-11-21T13:25:20,753 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T13:25:20,753 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T13:25:20,754 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T13:25:20,755 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3fb1b1f418508c88685394456e36ab0b, UNASSIGN}] 2024-11-21T13:25:20,756 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3fb1b1f418508c88685394456e36ab0b, UNASSIGN 2024-11-21T13:25:20,757 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=3fb1b1f418508c88685394456e36ab0b, regionState=CLOSING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:20,758 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=86, ppid=85, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=3fb1b1f418508c88685394456e36ab0b, UNASSIGN because future has completed 2024-11-21T13:25:20,758 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T13:25:20,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:25:20,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/92e6f855b8674fd5b9bfc9509acb5a68 2024-11-21T13:25:20,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/a86c743e521c42e98dae44728a243f7d is 50, key is test_row_0/B:col10/1732195520369/Put/seqid=0 2024-11-21T13:25:20,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742188_1364 (size=12301) 2024-11-21T13:25:20,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T13:25:20,910 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(122): Close 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:20,910 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T13:25:20,911 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1722): Closing 3fb1b1f418508c88685394456e36ab0b, disabling compactions & flushes 2024-11-21T13:25:20,911 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T13:25:21,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/a86c743e521c42e98dae44728a243f7d 2024-11-21T13:25:21,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/e5938f4b36064ce9b6f7660acf6b05bf is 50, key is test_row_0/C:col10/1732195520369/Put/seqid=0 2024-11-21T13:25:21,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742189_1365 (size=12301) 2024-11-21T13:25:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T13:25:21,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/e5938f4b36064ce9b6f7660acf6b05bf 2024-11-21T13:25:21,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/92e6f855b8674fd5b9bfc9509acb5a68 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/92e6f855b8674fd5b9bfc9509acb5a68 2024-11-21T13:25:21,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/92e6f855b8674fd5b9bfc9509acb5a68, entries=150, sequenceid=448, filesize=12.0 K 2024-11-21T13:25:21,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/a86c743e521c42e98dae44728a243f7d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/a86c743e521c42e98dae44728a243f7d 2024-11-21T13:25:21,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/a86c743e521c42e98dae44728a243f7d, entries=150, sequenceid=448, filesize=12.0 K 2024-11-21T13:25:21,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/e5938f4b36064ce9b6f7660acf6b05bf as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/e5938f4b36064ce9b6f7660acf6b05bf 2024-11-21T13:25:21,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/e5938f4b36064ce9b6f7660acf6b05bf, entries=150, sequenceid=448, filesize=12.0 K 2024-11-21T13:25:21,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=13.42 KB/13740 for 3fb1b1f418508c88685394456e36ab0b in 1265ms, sequenceid=448, compaction requested=true 2024-11-21T13:25:21,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 3fb1b1f418508c88685394456e36ab0b: 2024-11-21T13:25:21,636 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:21,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:21,636 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:21,636 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. because compaction request was cancelled 2024-11-21T13:25:21,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:21,636 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:A 2024-11-21T13:25:21,636 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. after waiting 0 ms 2024-11-21T13:25:21,636 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:21,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:B, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:21,636 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. because compaction request was cancelled 2024-11-21T13:25:21,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:21,636 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:B 2024-11-21T13:25:21,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 3fb1b1f418508c88685394456e36ab0b:C, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:21,636 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. because compaction request was cancelled 2024-11-21T13:25:21,636 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(2902): Flushing 3fb1b1f418508c88685394456e36ab0b 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-21T13:25:21,636 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 3fb1b1f418508c88685394456e36ab0b:C 2024-11-21T13:25:21,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:21,640 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/2e9219dc6173455e88cea96b83ea7ec2 is 50, key is test_row_1/A:col10/1732195520490/Put/seqid=0 2024-11-21T13:25:21,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742190_1366 (size=9857) 2024-11-21T13:25:21,645 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/2e9219dc6173455e88cea96b83ea7ec2 2024-11-21T13:25:21,652 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/d51494f776c842f88b2e5ff318ffafdf is 50, key is test_row_1/B:col10/1732195520490/Put/seqid=0 2024-11-21T13:25:21,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742191_1367 (size=9857) 2024-11-21T13:25:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T13:25:22,057 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/d51494f776c842f88b2e5ff318ffafdf 2024-11-21T13:25:22,068 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/4a093cfacccb4c76b30b429e4df4ca5e is 50, key is test_row_1/C:col10/1732195520490/Put/seqid=0 2024-11-21T13:25:22,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742192_1368 (size=9857) 2024-11-21T13:25:22,474 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=453 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/4a093cfacccb4c76b30b429e4df4ca5e 2024-11-21T13:25:22,484 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/A/2e9219dc6173455e88cea96b83ea7ec2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/2e9219dc6173455e88cea96b83ea7ec2 2024-11-21T13:25:22,488 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/2e9219dc6173455e88cea96b83ea7ec2, entries=100, sequenceid=453, filesize=9.6 K 2024-11-21T13:25:22,489 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/B/d51494f776c842f88b2e5ff318ffafdf as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/d51494f776c842f88b2e5ff318ffafdf 2024-11-21T13:25:22,493 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/d51494f776c842f88b2e5ff318ffafdf, entries=100, sequenceid=453, filesize=9.6 K 2024-11-21T13:25:22,494 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/.tmp/C/4a093cfacccb4c76b30b429e4df4ca5e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/4a093cfacccb4c76b30b429e4df4ca5e 2024-11-21T13:25:22,497 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/4a093cfacccb4c76b30b429e4df4ca5e, entries=100, sequenceid=453, filesize=9.6 K 2024-11-21T13:25:22,498 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 3fb1b1f418508c88685394456e36ab0b in 862ms, sequenceid=453, compaction requested=true 2024-11-21T13:25:22,498 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/ce0db2de02834cb5ae72b1010d94a484, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/caf1e6475d0c432086f041b9db2dad21, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/921b990c38cb4b919b76f134b018a1eb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/cc5c2eea274f4703ba7e1ec82a8a5cc2, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/447827fb577b49debb5e0fec26506a61, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/40558743d8e2497e991f9df968888af6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/3f9d8459846a42e789cc2d0d17154153, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/16862ba94d294226a98e13ec13abcec2, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/6923732cccc646c1b933c190f5a0e5f5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/68305233b0ca4f428e94a2652b66fac8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/8079fea0de2a4166b6a7c0616d9a4ebf, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/1daf35cb015f4a95a30972f6fe723540, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/11137f04db3645f1ad4417d702385f4c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/52957e4e35144d3ebcad805637f6c7d3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/749910f0e34f40c695a8672c8740be1c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/159bba570d344bf18675c2e0366c2f6c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/b7d496252da846a996f74800bea5ce45, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/e24e9a8f3fc34fa7b17698291bb6e7a1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/24308220dcbb4710aaaed91bb4f31682, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/06f59e49a061498a924703921c6525f9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/128f98c347ac436bb267683e43e93edc, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/98713d13f929467e80ccb85d120189ef, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/06e636cdb1cd4f8f8e94c364e374dd27, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/7055744dae6943fca9064ffe3d4c6081, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/427dbcb6b95949c0aeac843a90acdaff, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/cf1aa2e35e80448996d3ce22b90de903, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/33c16d701fb1413f8de554a3f7837c63, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/84be355a1c494e3480bdfe0a3efddb7c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/13e998b2532f45dfaf301217385a14b7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/430a97e2fc20483fad15947d1a41c12b] to archive 2024-11-21T13:25:22,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:25:22,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/ce0db2de02834cb5ae72b1010d94a484 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/ce0db2de02834cb5ae72b1010d94a484 2024-11-21T13:25:22,502 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/caf1e6475d0c432086f041b9db2dad21 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/caf1e6475d0c432086f041b9db2dad21 2024-11-21T13:25:22,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/921b990c38cb4b919b76f134b018a1eb to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/921b990c38cb4b919b76f134b018a1eb 2024-11-21T13:25:22,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/cc5c2eea274f4703ba7e1ec82a8a5cc2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/cc5c2eea274f4703ba7e1ec82a8a5cc2 2024-11-21T13:25:22,505 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/447827fb577b49debb5e0fec26506a61 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/447827fb577b49debb5e0fec26506a61 2024-11-21T13:25:22,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/40558743d8e2497e991f9df968888af6 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/40558743d8e2497e991f9df968888af6 2024-11-21T13:25:22,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/3f9d8459846a42e789cc2d0d17154153 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/3f9d8459846a42e789cc2d0d17154153 2024-11-21T13:25:22,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/16862ba94d294226a98e13ec13abcec2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/16862ba94d294226a98e13ec13abcec2 2024-11-21T13:25:22,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/6923732cccc646c1b933c190f5a0e5f5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/6923732cccc646c1b933c190f5a0e5f5 2024-11-21T13:25:22,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/68305233b0ca4f428e94a2652b66fac8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/68305233b0ca4f428e94a2652b66fac8 2024-11-21T13:25:22,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/8079fea0de2a4166b6a7c0616d9a4ebf to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/8079fea0de2a4166b6a7c0616d9a4ebf 2024-11-21T13:25:22,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/1daf35cb015f4a95a30972f6fe723540 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/1daf35cb015f4a95a30972f6fe723540 2024-11-21T13:25:22,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/11137f04db3645f1ad4417d702385f4c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/11137f04db3645f1ad4417d702385f4c 2024-11-21T13:25:22,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/52957e4e35144d3ebcad805637f6c7d3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/52957e4e35144d3ebcad805637f6c7d3 2024-11-21T13:25:22,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/749910f0e34f40c695a8672c8740be1c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/749910f0e34f40c695a8672c8740be1c 2024-11-21T13:25:22,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/159bba570d344bf18675c2e0366c2f6c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/159bba570d344bf18675c2e0366c2f6c 2024-11-21T13:25:22,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/b7d496252da846a996f74800bea5ce45 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/b7d496252da846a996f74800bea5ce45 2024-11-21T13:25:22,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/e24e9a8f3fc34fa7b17698291bb6e7a1 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/e24e9a8f3fc34fa7b17698291bb6e7a1 2024-11-21T13:25:22,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/24308220dcbb4710aaaed91bb4f31682 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/24308220dcbb4710aaaed91bb4f31682 2024-11-21T13:25:22,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/06f59e49a061498a924703921c6525f9 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/06f59e49a061498a924703921c6525f9 2024-11-21T13:25:22,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/128f98c347ac436bb267683e43e93edc to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/128f98c347ac436bb267683e43e93edc 2024-11-21T13:25:22,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/98713d13f929467e80ccb85d120189ef to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/98713d13f929467e80ccb85d120189ef 2024-11-21T13:25:22,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/06e636cdb1cd4f8f8e94c364e374dd27 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/06e636cdb1cd4f8f8e94c364e374dd27 2024-11-21T13:25:22,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/7055744dae6943fca9064ffe3d4c6081 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/7055744dae6943fca9064ffe3d4c6081 2024-11-21T13:25:22,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/427dbcb6b95949c0aeac843a90acdaff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/427dbcb6b95949c0aeac843a90acdaff 2024-11-21T13:25:22,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/cf1aa2e35e80448996d3ce22b90de903 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/cf1aa2e35e80448996d3ce22b90de903 2024-11-21T13:25:22,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/33c16d701fb1413f8de554a3f7837c63 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/33c16d701fb1413f8de554a3f7837c63 2024-11-21T13:25:22,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/84be355a1c494e3480bdfe0a3efddb7c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/84be355a1c494e3480bdfe0a3efddb7c 2024-11-21T13:25:22,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/13e998b2532f45dfaf301217385a14b7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/13e998b2532f45dfaf301217385a14b7 2024-11-21T13:25:22,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/430a97e2fc20483fad15947d1a41c12b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/430a97e2fc20483fad15947d1a41c12b 2024-11-21T13:25:22,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/74a66cab894b47798422ba716040aad8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9da6a8f6337b4c7dbee0675037fe0e61, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/1551f596220747fa9cb425c594e54d4c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/10087a4f5bff4757ad0e60dcec815b7b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/c459800bc7354c17abbe4c9315f2e1f7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/dc46f6b2d02b4def98df00aa8f047e54, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/46f0ed1fa6b5437b89c04a84239a0d62, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/95641fb7f3b04f31bda16033ce526b0a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/7191f46413274c27b3d2dc8b6fa5b854, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/e17ccd236205402fadce1423e6664f1c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/5e8fe6c8555d412fa82de61c7a150374, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/145ab3e01bc943708a0b7920c2379e8d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9ea2fdcd16a04b2d9edd47bc52a3fb18, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/3bcb670f499f4787a76c78490891250d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/644b2b4da29a4d23abbc9efadcdfd961, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/575701ee8b0d4b3d8317b5053bcced55, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/ac637b07b3fe478a8ab39186baa2780d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/d0a5e6d49ca14c9db224441ca1019be0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/4bc93b65c98d4bea9fea53088ec48a4e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9e9e948019c04a6fa2f9f1d79d83afae, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/8599f8f7c819423d89dc1dc64b7b9c01, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/4f77c60a5b8640f4895af3d222276cc1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/16f35a6844c2416f939b540c07b13cd1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/651a55696af148d58543ee48f6ef3f5c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/c00f657f5ac846b9b313b1ad1a37acc4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/594722d1898340168de487c6b405b892, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/35cded11313b4ffa9fad260361eea300, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/e203d6ac615444b085c9badb504956aa, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/22ab53e7fa0c4891aad1f37177d62df8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/6eb98971950a4f2e991a934141c29c6c] to archive 2024-11-21T13:25:22,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:25:22,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/74a66cab894b47798422ba716040aad8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/74a66cab894b47798422ba716040aad8 2024-11-21T13:25:22,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9da6a8f6337b4c7dbee0675037fe0e61 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9da6a8f6337b4c7dbee0675037fe0e61 2024-11-21T13:25:22,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/1551f596220747fa9cb425c594e54d4c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/1551f596220747fa9cb425c594e54d4c 2024-11-21T13:25:22,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/10087a4f5bff4757ad0e60dcec815b7b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/10087a4f5bff4757ad0e60dcec815b7b 2024-11-21T13:25:22,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/c459800bc7354c17abbe4c9315f2e1f7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/c459800bc7354c17abbe4c9315f2e1f7 2024-11-21T13:25:22,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/dc46f6b2d02b4def98df00aa8f047e54 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/dc46f6b2d02b4def98df00aa8f047e54 2024-11-21T13:25:22,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/46f0ed1fa6b5437b89c04a84239a0d62 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/46f0ed1fa6b5437b89c04a84239a0d62 2024-11-21T13:25:22,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/95641fb7f3b04f31bda16033ce526b0a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/95641fb7f3b04f31bda16033ce526b0a 2024-11-21T13:25:22,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/7191f46413274c27b3d2dc8b6fa5b854 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/7191f46413274c27b3d2dc8b6fa5b854 2024-11-21T13:25:22,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/e17ccd236205402fadce1423e6664f1c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/e17ccd236205402fadce1423e6664f1c 2024-11-21T13:25:22,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/5e8fe6c8555d412fa82de61c7a150374 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/5e8fe6c8555d412fa82de61c7a150374 2024-11-21T13:25:22,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/145ab3e01bc943708a0b7920c2379e8d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/145ab3e01bc943708a0b7920c2379e8d 2024-11-21T13:25:22,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9ea2fdcd16a04b2d9edd47bc52a3fb18 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9ea2fdcd16a04b2d9edd47bc52a3fb18 2024-11-21T13:25:22,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/3bcb670f499f4787a76c78490891250d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/3bcb670f499f4787a76c78490891250d 2024-11-21T13:25:22,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/644b2b4da29a4d23abbc9efadcdfd961 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/644b2b4da29a4d23abbc9efadcdfd961 2024-11-21T13:25:22,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/575701ee8b0d4b3d8317b5053bcced55 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/575701ee8b0d4b3d8317b5053bcced55 2024-11-21T13:25:22,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/ac637b07b3fe478a8ab39186baa2780d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/ac637b07b3fe478a8ab39186baa2780d 2024-11-21T13:25:22,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/d0a5e6d49ca14c9db224441ca1019be0 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/d0a5e6d49ca14c9db224441ca1019be0 2024-11-21T13:25:22,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/4bc93b65c98d4bea9fea53088ec48a4e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/4bc93b65c98d4bea9fea53088ec48a4e 2024-11-21T13:25:22,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9e9e948019c04a6fa2f9f1d79d83afae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/9e9e948019c04a6fa2f9f1d79d83afae 2024-11-21T13:25:22,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/8599f8f7c819423d89dc1dc64b7b9c01 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/8599f8f7c819423d89dc1dc64b7b9c01 2024-11-21T13:25:22,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/4f77c60a5b8640f4895af3d222276cc1 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/4f77c60a5b8640f4895af3d222276cc1 2024-11-21T13:25:22,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/16f35a6844c2416f939b540c07b13cd1 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/16f35a6844c2416f939b540c07b13cd1 2024-11-21T13:25:22,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/651a55696af148d58543ee48f6ef3f5c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/651a55696af148d58543ee48f6ef3f5c 2024-11-21T13:25:22,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/c00f657f5ac846b9b313b1ad1a37acc4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/c00f657f5ac846b9b313b1ad1a37acc4 2024-11-21T13:25:22,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/594722d1898340168de487c6b405b892 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/594722d1898340168de487c6b405b892 2024-11-21T13:25:22,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/35cded11313b4ffa9fad260361eea300 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/35cded11313b4ffa9fad260361eea300 2024-11-21T13:25:22,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/e203d6ac615444b085c9badb504956aa to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/e203d6ac615444b085c9badb504956aa 2024-11-21T13:25:22,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/22ab53e7fa0c4891aad1f37177d62df8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/22ab53e7fa0c4891aad1f37177d62df8 2024-11-21T13:25:22,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/6eb98971950a4f2e991a934141c29c6c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/6eb98971950a4f2e991a934141c29c6c 2024-11-21T13:25:22,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/819411b05efd401182dab0dcfd69b54a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/10e6a4a13f494613b607f4f75b2e5c67, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/f55e084c0df949dbbbcf0c076bc04b40, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/3eeede3b66bc4005aba9ff319fb37092, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/599bce3fcaea4be389dbfddaa3dd461c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/2f44612a05134d43bacc433606fde55e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/481b4ada78f0439aa4f74bd16c5e00be, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/6f86adee467c421e9c7dd29719c2784f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b85e769a31184bd896dd22f349d31d61, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/828b0af726924a2fbf2c7057657ba0a9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/2648c34f58b04c15a0811153135b7a75, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/c89dd676f6a746e5b57b6b1ff25c511c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/f4ad98108dd94fd6b642bf43ace39fae, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/29ffe480d35c401994863788240479f0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/85ebe130ff874631909a859c1340f709, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b5dbc203fccf49aca97b7bd576fa048e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/a9fdf2edcfa141ddbe861922de379723, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/553b8486dced43888fa53c7f576ed249, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b553425e1408469287ee35daae3fc794, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b940453fbd7e4541914ff52fc5ca0eab, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/26f48451fe05441d9e3575777da6e55b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/c03271f454204da286995014b5060d64, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/220fdcfa2624431db02647820274e870, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/8d1b015ee30047d2afb3528786acc04b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/9b3f06f05e734b8ea363830aed553df2, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/d8ce758d10e7454c86379ee309a2b5e7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/8142e9b644354e7fa41c7271f6692d75, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/65eaff3bfe1141e48491c45d3066e933, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/001ded2db28849828ca2ed2181900ead, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/34b25a1204dc417c9d39ae18d1e8e36f] to archive 2024-11-21T13:25:22,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:25:22,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/819411b05efd401182dab0dcfd69b54a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/819411b05efd401182dab0dcfd69b54a 2024-11-21T13:25:22,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/10e6a4a13f494613b607f4f75b2e5c67 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/10e6a4a13f494613b607f4f75b2e5c67 2024-11-21T13:25:22,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/f55e084c0df949dbbbcf0c076bc04b40 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/f55e084c0df949dbbbcf0c076bc04b40 2024-11-21T13:25:22,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/3eeede3b66bc4005aba9ff319fb37092 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/3eeede3b66bc4005aba9ff319fb37092 2024-11-21T13:25:22,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/599bce3fcaea4be389dbfddaa3dd461c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/599bce3fcaea4be389dbfddaa3dd461c 2024-11-21T13:25:22,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/2f44612a05134d43bacc433606fde55e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/2f44612a05134d43bacc433606fde55e 2024-11-21T13:25:22,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/481b4ada78f0439aa4f74bd16c5e00be to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/481b4ada78f0439aa4f74bd16c5e00be 2024-11-21T13:25:22,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/6f86adee467c421e9c7dd29719c2784f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/6f86adee467c421e9c7dd29719c2784f 2024-11-21T13:25:22,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b85e769a31184bd896dd22f349d31d61 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b85e769a31184bd896dd22f349d31d61 2024-11-21T13:25:22,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/828b0af726924a2fbf2c7057657ba0a9 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/828b0af726924a2fbf2c7057657ba0a9 2024-11-21T13:25:22,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/2648c34f58b04c15a0811153135b7a75 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/2648c34f58b04c15a0811153135b7a75 2024-11-21T13:25:22,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/c89dd676f6a746e5b57b6b1ff25c511c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/c89dd676f6a746e5b57b6b1ff25c511c 2024-11-21T13:25:22,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/f4ad98108dd94fd6b642bf43ace39fae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/f4ad98108dd94fd6b642bf43ace39fae 2024-11-21T13:25:22,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/29ffe480d35c401994863788240479f0 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/29ffe480d35c401994863788240479f0 2024-11-21T13:25:22,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/85ebe130ff874631909a859c1340f709 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/85ebe130ff874631909a859c1340f709 2024-11-21T13:25:22,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b5dbc203fccf49aca97b7bd576fa048e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b5dbc203fccf49aca97b7bd576fa048e 2024-11-21T13:25:22,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/a9fdf2edcfa141ddbe861922de379723 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/a9fdf2edcfa141ddbe861922de379723 2024-11-21T13:25:22,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/553b8486dced43888fa53c7f576ed249 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/553b8486dced43888fa53c7f576ed249 2024-11-21T13:25:22,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b553425e1408469287ee35daae3fc794 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b553425e1408469287ee35daae3fc794 2024-11-21T13:25:22,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b940453fbd7e4541914ff52fc5ca0eab to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b940453fbd7e4541914ff52fc5ca0eab 2024-11-21T13:25:22,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/26f48451fe05441d9e3575777da6e55b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/26f48451fe05441d9e3575777da6e55b 2024-11-21T13:25:22,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/c03271f454204da286995014b5060d64 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/c03271f454204da286995014b5060d64 2024-11-21T13:25:22,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/220fdcfa2624431db02647820274e870 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/220fdcfa2624431db02647820274e870 2024-11-21T13:25:22,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/8d1b015ee30047d2afb3528786acc04b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/8d1b015ee30047d2afb3528786acc04b 2024-11-21T13:25:22,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/9b3f06f05e734b8ea363830aed553df2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/9b3f06f05e734b8ea363830aed553df2 2024-11-21T13:25:22,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/d8ce758d10e7454c86379ee309a2b5e7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/d8ce758d10e7454c86379ee309a2b5e7 2024-11-21T13:25:22,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/8142e9b644354e7fa41c7271f6692d75 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/8142e9b644354e7fa41c7271f6692d75 2024-11-21T13:25:22,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/65eaff3bfe1141e48491c45d3066e933 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/65eaff3bfe1141e48491c45d3066e933 2024-11-21T13:25:22,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/001ded2db28849828ca2ed2181900ead to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/001ded2db28849828ca2ed2181900ead 2024-11-21T13:25:22,574 DEBUG [StoreCloser-TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/34b25a1204dc417c9d39ae18d1e8e36f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/34b25a1204dc417c9d39ae18d1e8e36f 2024-11-21T13:25:22,578 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/recovered.edits/456.seqid, newMaxSeqId=456, maxSeqId=1 2024-11-21T13:25:22,578 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. 2024-11-21T13:25:22,578 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] regionserver.HRegion(1676): Region close journal for 3fb1b1f418508c88685394456e36ab0b: Waiting for close lock at 1732195520911Running coprocessor pre-close hooks at 1732195520911Disabling compacts and flushes for region at 1732195520911Disabling writes for close at 1732195521636 (+725 ms)Obtaining lock to block concurrent updates at 1732195521636Preparing flush snapshotting stores in 3fb1b1f418508c88685394456e36ab0b at 1732195521636Finished memstore snapshotting TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732195521636Flushing stores of TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b. at 1732195521637 (+1 ms)Flushing 3fb1b1f418508c88685394456e36ab0b/A: creating writer at 1732195521637Flushing 3fb1b1f418508c88685394456e36ab0b/A: appending metadata at 1732195521639 (+2 ms)Flushing 3fb1b1f418508c88685394456e36ab0b/A: closing flushed file at 1732195521639Flushing 3fb1b1f418508c88685394456e36ab0b/B: creating writer at 1732195521649 (+10 ms)Flushing 3fb1b1f418508c88685394456e36ab0b/B: appending metadata at 1732195521652 (+3 ms)Flushing 3fb1b1f418508c88685394456e36ab0b/B: closing flushed file at 1732195521652Flushing 3fb1b1f418508c88685394456e36ab0b/C: creating writer at 1732195522064 (+412 ms)Flushing 3fb1b1f418508c88685394456e36ab0b/C: appending metadata at 1732195522068 (+4 ms)Flushing 3fb1b1f418508c88685394456e36ab0b/C: closing flushed file at 1732195522068Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1cc43539: reopening flushed file at 1732195522483 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7b05a1a5: reopening flushed file at 1732195522488 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2faf265a: reopening flushed file at 1732195522493 (+5 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 3fb1b1f418508c88685394456e36ab0b in 862ms, sequenceid=453, compaction requested=true at 1732195522498 (+5 ms)Writing region close event to WAL at 1732195522576 (+78 ms)Running coprocessor post-close hooks at 1732195522578 (+2 ms)Closed at 1732195522578 2024-11-21T13:25:22,580 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=87}] handler.UnassignRegionHandler(157): Closed 3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:22,581 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=86 updating hbase:meta row=3fb1b1f418508c88685394456e36ab0b, regionState=CLOSED 2024-11-21T13:25:22,582 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=87, ppid=86, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:25:22,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-21T13:25:22,585 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseRegionProcedure 3fb1b1f418508c88685394456e36ab0b, server=4fe6efa1e611,33573,1732195430851 in 1.8250 sec 2024-11-21T13:25:22,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-21T13:25:22,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=3fb1b1f418508c88685394456e36ab0b, UNASSIGN in 1.8300 sec 2024-11-21T13:25:22,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-21T13:25:22,590 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8340 sec 2024-11-21T13:25:22,591 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195522591"}]},"ts":"1732195522591"} 2024-11-21T13:25:22,593 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T13:25:22,593 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T13:25:22,595 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8480 sec 2024-11-21T13:25:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-21T13:25:22,900 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:22,902 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T13:25:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:22,907 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=88, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:22,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T13:25:22,909 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=88, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:22,912 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:22,916 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/recovered.edits] 2024-11-21T13:25:22,922 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/2e9219dc6173455e88cea96b83ea7ec2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/2e9219dc6173455e88cea96b83ea7ec2 2024-11-21T13:25:22,924 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/6df564f859684feb9ac5a7241852f6e5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/6df564f859684feb9ac5a7241852f6e5 2024-11-21T13:25:22,927 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/92e6f855b8674fd5b9bfc9509acb5a68 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/92e6f855b8674fd5b9bfc9509acb5a68 2024-11-21T13:25:22,929 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/b5c0bc69fddc4698a29dbf6579b4a55e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/A/b5c0bc69fddc4698a29dbf6579b4a55e 2024-11-21T13:25:22,934 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/35a9333c60be46c2afc7fbc1a39bd99d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/35a9333c60be46c2afc7fbc1a39bd99d 2024-11-21T13:25:22,937 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/a86c743e521c42e98dae44728a243f7d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/a86c743e521c42e98dae44728a243f7d 2024-11-21T13:25:22,939 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/d51494f776c842f88b2e5ff318ffafdf to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/d51494f776c842f88b2e5ff318ffafdf 2024-11-21T13:25:22,942 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/f1a633c699db47d9862d6bd3591bbfcd to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/B/f1a633c699db47d9862d6bd3591bbfcd 2024-11-21T13:25:22,946 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/0b9351315a7f41e981546fc5b5a92ea4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/0b9351315a7f41e981546fc5b5a92ea4 2024-11-21T13:25:22,949 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/4a093cfacccb4c76b30b429e4df4ca5e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/4a093cfacccb4c76b30b429e4df4ca5e 2024-11-21T13:25:22,952 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b4a9a49f860946a1beb65812cd7e6d39 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/b4a9a49f860946a1beb65812cd7e6d39 2024-11-21T13:25:22,955 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/e5938f4b36064ce9b6f7660acf6b05bf to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/C/e5938f4b36064ce9b6f7660acf6b05bf 2024-11-21T13:25:22,961 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/recovered.edits/456.seqid to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b/recovered.edits/456.seqid 2024-11-21T13:25:22,962 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/3fb1b1f418508c88685394456e36ab0b 2024-11-21T13:25:22,962 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T13:25:22,964 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=88, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:22,968 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T13:25:22,970 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T13:25:22,971 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=88, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:22,971 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T13:25:22,971 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732195522971"}]},"ts":"9223372036854775807"} 2024-11-21T13:25:22,973 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T13:25:22,973 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3fb1b1f418508c88685394456e36ab0b, NAME => 'TestAcidGuarantees,,1732195497944.3fb1b1f418508c88685394456e36ab0b.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T13:25:22,973 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T13:25:22,973 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732195522973"}]},"ts":"9223372036854775807"} 2024-11-21T13:25:22,975 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T13:25:22,975 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=88, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:22,976 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 73 msec 2024-11-21T13:25:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=88 2024-11-21T13:25:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T13:25:23,020 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:23,030 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=241 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_572311665_22 at /127.0.0.1:35532 [Waiting for operation #167] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1640571641_22 at /127.0.0.1:35546 [Waiting for operation #166] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1640571641_22 at /127.0.0.1:33990 [Waiting for operation #301] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=508 (was 495) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2397 (was 3165) 2024-11-21T13:25:23,039 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=508, ProcessCount=11, AvailableMemoryMB=2396 2024-11-21T13:25:23,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T13:25:23,041 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T13:25:23,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:23,043 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T13:25:23,043 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:23,043 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 89 2024-11-21T13:25:23,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T13:25:23,044 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T13:25:23,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742193_1369 (size=959) 2024-11-21T13:25:23,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T13:25:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T13:25:23,456 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48 2024-11-21T13:25:23,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742194_1370 (size=53) 2024-11-21T13:25:23,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T13:25:23,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:25:23,869 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6e6e9ba8386d0e037fdad07b55820eae, disabling compactions & flushes 2024-11-21T13:25:23,869 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:23,870 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:23,870 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. after waiting 0 ms 2024-11-21T13:25:23,870 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:23,870 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:23,870 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6e6e9ba8386d0e037fdad07b55820eae: Waiting for close lock at 1732195523869Disabling compacts and flushes for region at 1732195523869Disabling writes for close at 1732195523870 (+1 ms)Writing region close event to WAL at 1732195523870Closed at 1732195523870 2024-11-21T13:25:23,873 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T13:25:23,874 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732195523873"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732195523873"}]},"ts":"1732195523873"} 2024-11-21T13:25:23,876 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T13:25:23,877 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T13:25:23,878 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195523877"}]},"ts":"1732195523877"} 2024-11-21T13:25:23,879 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T13:25:23,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, ASSIGN}] 2024-11-21T13:25:23,881 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, ASSIGN 2024-11-21T13:25:23,882 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, ASSIGN; state=OFFLINE, location=4fe6efa1e611,33573,1732195430851; forceNewPlan=false, retain=false 2024-11-21T13:25:24,033 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=6e6e9ba8386d0e037fdad07b55820eae, regionState=OPENING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:24,037 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=90, ppid=89, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, ASSIGN because future has completed 2024-11-21T13:25:24,038 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:25:24,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T13:25:24,200 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:24,200 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} 2024-11-21T13:25:24,201 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:24,201 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:25:24,202 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7794): checking encryption for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:24,202 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(7797): checking classloading for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:24,204 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:24,206 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e6e9ba8386d0e037fdad07b55820eae columnFamilyName A 2024-11-21T13:25:24,206 DEBUG [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:24,206 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] regionserver.HStore(327): Store=6e6e9ba8386d0e037fdad07b55820eae/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:25:24,206 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:24,208 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e6e9ba8386d0e037fdad07b55820eae columnFamilyName B 2024-11-21T13:25:24,208 DEBUG [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:24,208 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] regionserver.HStore(327): Store=6e6e9ba8386d0e037fdad07b55820eae/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:25:24,208 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:24,209 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e6e9ba8386d0e037fdad07b55820eae columnFamilyName C 2024-11-21T13:25:24,210 DEBUG [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:24,210 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] regionserver.HStore(327): Store=6e6e9ba8386d0e037fdad07b55820eae/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:25:24,210 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1038): replaying wal for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:24,211 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:24,211 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:24,212 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1048): stopping wal replay for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:24,212 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1060): Cleaning up temporary data for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:24,213 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T13:25:24,214 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1093): writing seq id for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:24,216 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T13:25:24,217 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1114): Opened 6e6e9ba8386d0e037fdad07b55820eae; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71034417, jitterRate=0.05849529802799225}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T13:25:24,217 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:24,217 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegion(1006): Region open journal for 6e6e9ba8386d0e037fdad07b55820eae: Running coprocessor pre-open hook at 1732195524202Writing region info on filesystem at 1732195524202Initializing all the Stores at 1732195524203 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195524204 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195524204Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195524204Cleaning up temporary data from old regions at 1732195524212 (+8 ms)Running coprocessor post-open hooks at 1732195524217 (+5 ms)Region opened successfully at 1732195524217 2024-11-21T13:25:24,218 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., pid=91, masterSystemTime=1732195524193 2024-11-21T13:25:24,220 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:24,220 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=91}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:24,220 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=90 updating hbase:meta row=6e6e9ba8386d0e037fdad07b55820eae, regionState=OPEN, openSeqNum=2, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:24,222 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=91, ppid=90, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:25:24,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-21T13:25:24,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 in 185 msec 2024-11-21T13:25:24,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-21T13:25:24,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, ASSIGN in 345 msec 2024-11-21T13:25:24,227 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T13:25:24,227 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195524227"}]},"ts":"1732195524227"} 2024-11-21T13:25:24,228 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T13:25:24,229 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=89, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T13:25:24,230 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-11-21T13:25:25,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-21T13:25:25,189 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:25,190 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3455134, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:25,190 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:25,191 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:25,191 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5cf73f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:25,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:25,193 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77cfb6c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:25,194 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:25,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:25,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:25,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54984, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:25,196 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T13:25:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 4fe6efa1e611,46031,1732195430123 2024-11-21T13:25:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33b80947 2024-11-21T13:25:25,197 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T13:25:25,198 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45848, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T13:25:25,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T13:25:25,200 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T13:25:25,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:25,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742195_1371 (size=995) 2024-11-21T13:25:25,611 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T13:25:25,611 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T13:25:25,613 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T13:25:25,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, REOPEN/MOVE}] 2024-11-21T13:25:25,616 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, REOPEN/MOVE 2024-11-21T13:25:25,617 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=6e6e9ba8386d0e037fdad07b55820eae, regionState=CLOSING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:25,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, REOPEN/MOVE because future has completed 2024-11-21T13:25:25,619 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T13:25:25,619 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:25:25,771 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(122): Close 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:25,771 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T13:25:25,771 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1722): Closing 6e6e9ba8386d0e037fdad07b55820eae, disabling compactions & flushes 2024-11-21T13:25:25,771 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:25,771 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:25,771 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. after waiting 0 ms 2024-11-21T13:25:25,771 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:25,776 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T13:25:25,776 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:25,776 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegion(1676): Region close journal for 6e6e9ba8386d0e037fdad07b55820eae: Waiting for close lock at 1732195525771Running coprocessor pre-close hooks at 1732195525771Disabling compacts and flushes for region at 1732195525771Disabling writes for close at 1732195525771Writing region close event to WAL at 1732195525772 (+1 ms)Running coprocessor post-close hooks at 1732195525776 (+4 ms)Closed at 1732195525776 2024-11-21T13:25:25,776 WARN [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] regionserver.HRegionServer(3299): Not adding moved region record: 6e6e9ba8386d0e037fdad07b55820eae to self. 2024-11-21T13:25:25,778 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=95}] handler.UnassignRegionHandler(157): Closed 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:25,779 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=6e6e9ba8386d0e037fdad07b55820eae, regionState=CLOSED 2024-11-21T13:25:25,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=95, ppid=94, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:25:25,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-21T13:25:25,783 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; CloseRegionProcedure 6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 in 162 msec 2024-11-21T13:25:25,784 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, REOPEN/MOVE; state=CLOSED, location=4fe6efa1e611,33573,1732195430851; forceNewPlan=false, retain=true 2024-11-21T13:25:25,934 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=6e6e9ba8386d0e037fdad07b55820eae, regionState=OPENING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:25,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=93, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, REOPEN/MOVE because future has completed 2024-11-21T13:25:25,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:25:26,091 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:26,091 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} 2024-11-21T13:25:26,092 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,092 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:25:26,092 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7794): checking encryption for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,092 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(7797): checking classloading for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,093 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,095 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e6e9ba8386d0e037fdad07b55820eae columnFamilyName A 2024-11-21T13:25:26,097 DEBUG [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:26,097 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] regionserver.HStore(327): Store=6e6e9ba8386d0e037fdad07b55820eae/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:25:26,097 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,098 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e6e9ba8386d0e037fdad07b55820eae columnFamilyName B 2024-11-21T13:25:26,098 DEBUG [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:26,099 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] regionserver.HStore(327): Store=6e6e9ba8386d0e037fdad07b55820eae/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:25:26,099 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,100 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e6e9ba8386d0e037fdad07b55820eae columnFamilyName C 2024-11-21T13:25:26,100 DEBUG [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:26,100 INFO [StoreOpener-6e6e9ba8386d0e037fdad07b55820eae-1 {}] regionserver.HStore(327): Store=6e6e9ba8386d0e037fdad07b55820eae/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:25:26,100 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1038): replaying wal for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,101 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,102 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,103 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1048): stopping wal replay for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,103 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1060): Cleaning up temporary data for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,103 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T13:25:26,104 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1093): writing seq id for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,105 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1114): Opened 6e6e9ba8386d0e037fdad07b55820eae; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63782707, jitterRate=-0.04956360161304474}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T13:25:26,105 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,106 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegion(1006): Region open journal for 6e6e9ba8386d0e037fdad07b55820eae: Running coprocessor pre-open hook at 1732195526092Writing region info on filesystem at 1732195526092Initializing all the Stores at 1732195526093 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195526093Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195526093Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195526093Cleaning up temporary data from old regions at 1732195526103 (+10 ms)Running coprocessor post-open hooks at 1732195526105 (+2 ms)Region opened successfully at 1732195526106 (+1 ms) 2024-11-21T13:25:26,106 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., pid=96, masterSystemTime=1732195526088 2024-11-21T13:25:26,108 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:26,109 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=96}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:26,109 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=94 updating hbase:meta row=6e6e9ba8386d0e037fdad07b55820eae, regionState=OPEN, openSeqNum=5, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,110 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=94, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:25:26,113 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=94 2024-11-21T13:25:26,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=94, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 in 175 msec 2024-11-21T13:25:26,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-21T13:25:26,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, REOPEN/MOVE in 497 msec 2024-11-21T13:25:26,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-21T13:25:26,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 503 msec 2024-11-21T13:25:26,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 918 msec 2024-11-21T13:25:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-11-21T13:25:26,121 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:26,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37260de2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,121 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:26,121 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:26,122 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:26,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:26,122 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:26,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bae22ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:26,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:26,123 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:26,124 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:26,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eacbed7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,125 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a1929e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,125 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:26,125 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:26,126 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:26,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68a27e44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:26,127 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:26,128 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:26,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cf99154, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b673756, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,130 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:26,130 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:26,131 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad40f02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:26,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:26,132 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45904, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:26,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32e0f659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7999da1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:26,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:26,134 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@790c29e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:26,135 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:26,136 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dcfffe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,137 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781a5df7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,137 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:26,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:26,138 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@995e8de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:26,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:26,140 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45928, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:26,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45d669f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4140c04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,142 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:26,142 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:26,143 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5325920c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:26,143 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:26,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:26,144 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:26,145 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ffd2331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@230cc9b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:26,149 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:26,150 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e08d069, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:26,150 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:26,151 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:26,152 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:26,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ad5780e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@122111e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,156 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:26,156 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:26,157 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:26,157 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c2adcfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:26,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:26,159 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:26,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@464c9465, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,160 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25dafdb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,160 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:26,160 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:26,161 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:26,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:26,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50ba0f60, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:26,162 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:26,163 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:26,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@385453c4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,168 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@362b1219, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,169 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:26,169 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:26,170 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f541e63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,170 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:26,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:26,172 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46020, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:26,172 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64558986, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:26,175 DEBUG [Thread-1678 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:26,175 DEBUG [Thread-1689 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:26,176 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:26,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:26,178 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:25:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:25:26,180 DEBUG [Thread-1680 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:26,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:26,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:26,183 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:26,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:25:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,194 DEBUG [Thread-1685 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:26,194 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:26,194 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T13:25:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:26,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:26,196 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:26,198 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:25:26,217 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-21T13:25:26,218 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T13:25:26,219 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:26,219 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:26,228 DEBUG [Thread-1676 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:26,229 DEBUG [Thread-1687 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:26,229 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:26,230 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:26,231 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:26,232 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55030, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:26,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:25:26,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:25:26,254 DEBUG [Thread-1691 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:26,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:26,255 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:26,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:26,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:25:26,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195586249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,267 DEBUG [Thread-1674 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:26,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:26,269 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:26,270 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,271 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,272 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:25:26,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,283 DEBUG [Thread-1672 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195586261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:26,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:26,286 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55074, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:26,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195586263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195586273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,287 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:25:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 1 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195586289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121fe074729d08544aebf67cdcb02ee4d92_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195526187/Put/seqid=0 2024-11-21T13:25:26,323 DEBUG [Thread-1683 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:26,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:26,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:26,325 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55078, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:26,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:25:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T13:25:26,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:26,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:26,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:26,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:26,377 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:26,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:26,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195586380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742196_1372 (size=12154) 2024-11-21T13:25:26,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,386 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:26,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,386 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,389 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121fe074729d08544aebf67cdcb02ee4d92_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fe074729d08544aebf67cdcb02ee4d92_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,392 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/8615e316bb964f34ad9fffb17ffb50f4, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:26,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/8615e316bb964f34ad9fffb17ffb50f4 is 175, key is test_row_0/A:col10/1732195526187/Put/seqid=0 2024-11-21T13:25:26,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195586390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195586399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195586399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195586399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742197_1373 (size=30955) 2024-11-21T13:25:26,418 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/8615e316bb964f34ad9fffb17ffb50f4 2024-11-21T13:25:26,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/bf31e3c5a5c945959e8a0619bced5cbd is 50, key is test_row_0/B:col10/1732195526187/Put/seqid=0 2024-11-21T13:25:26,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742198_1374 (size=12001) 2024-11-21T13:25:26,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/bf31e3c5a5c945959e8a0619bced5cbd 2024-11-21T13:25:26,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/b91c6f738fa0426ea82f3794fb8d0f4a is 50, key is test_row_0/C:col10/1732195526187/Put/seqid=0 2024-11-21T13:25:26,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:26,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:26,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:26,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:26,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:26,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:26,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742199_1375 (size=12001) 2024-11-21T13:25:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T13:25:26,542 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/b91c6f738fa0426ea82f3794fb8d0f4a 2024-11-21T13:25:26,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/8615e316bb964f34ad9fffb17ffb50f4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8615e316bb964f34ad9fffb17ffb50f4 2024-11-21T13:25:26,558 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8615e316bb964f34ad9fffb17ffb50f4, entries=150, sequenceid=16, filesize=30.2 K 2024-11-21T13:25:26,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/bf31e3c5a5c945959e8a0619bced5cbd as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bf31e3c5a5c945959e8a0619bced5cbd 2024-11-21T13:25:26,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bf31e3c5a5c945959e8a0619bced5cbd, entries=150, sequenceid=16, filesize=11.7 K 2024-11-21T13:25:26,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/b91c6f738fa0426ea82f3794fb8d0f4a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/b91c6f738fa0426ea82f3794fb8d0f4a 2024-11-21T13:25:26,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/b91c6f738fa0426ea82f3794fb8d0f4a, entries=150, sequenceid=16, filesize=11.7 K 2024-11-21T13:25:26,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 6e6e9ba8386d0e037fdad07b55820eae in 377ms, sequenceid=16, compaction requested=false 2024-11-21T13:25:26,572 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-21T13:25:26,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:26,602 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:26,602 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T13:25:26,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:26,629 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214961a43a0f324c5bb159cb4561b458ae_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195526243/Put/seqid=0 2024-11-21T13:25:26,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195586618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,632 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195586619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,633 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195586625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195586629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195586631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:26,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:26,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:26,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:26,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:26,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:26,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:26,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742200_1376 (size=14594) 2024-11-21T13:25:26,751 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195586739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195586739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,751 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195586739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195586750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195586750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:26,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:26,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:26,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:26,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:26,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T13:25:26,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195586960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195586960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195586960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195586969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195586969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:26,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:26,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:26,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:26,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:26,996 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:26,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,094 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:27,105 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411214961a43a0f324c5bb159cb4561b458ae_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214961a43a0f324c5bb159cb4561b458ae_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:27,106 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/f5cf700f1bd44acd856d83c118a9b4cb, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:27,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/f5cf700f1bd44acd856d83c118a9b4cb is 175, key is test_row_0/A:col10/1732195526243/Put/seqid=0 2024-11-21T13:25:27,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742201_1377 (size=39549) 2024-11-21T13:25:27,131 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=42, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/f5cf700f1bd44acd856d83c118a9b4cb 2024-11-21T13:25:27,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/f0dde327118f4ac089a53f3604bc27d7 is 50, key is test_row_0/B:col10/1732195526243/Put/seqid=0 2024-11-21T13:25:27,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:27,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:27,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:27,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:27,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742202_1378 (size=12001) 2024-11-21T13:25:27,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/f0dde327118f4ac089a53f3604bc27d7 2024-11-21T13:25:27,191 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/7c587dc0479740dc921213737a32e16a is 50, key is test_row_0/C:col10/1732195526243/Put/seqid=0 2024-11-21T13:25:27,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742203_1379 (size=12001) 2024-11-21T13:25:27,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=42 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/7c587dc0479740dc921213737a32e16a 2024-11-21T13:25:27,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/f5cf700f1bd44acd856d83c118a9b4cb as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/f5cf700f1bd44acd856d83c118a9b4cb 2024-11-21T13:25:27,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/f5cf700f1bd44acd856d83c118a9b4cb, entries=200, sequenceid=42, filesize=38.6 K 2024-11-21T13:25:27,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/f0dde327118f4ac089a53f3604bc27d7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/f0dde327118f4ac089a53f3604bc27d7 2024-11-21T13:25:27,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/f0dde327118f4ac089a53f3604bc27d7, entries=150, sequenceid=42, filesize=11.7 K 2024-11-21T13:25:27,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/7c587dc0479740dc921213737a32e16a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/7c587dc0479740dc921213737a32e16a 2024-11-21T13:25:27,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/7c587dc0479740dc921213737a32e16a, entries=150, sequenceid=42, filesize=11.7 K 2024-11-21T13:25:27,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6e6e9ba8386d0e037fdad07b55820eae in 631ms, sequenceid=42, compaction requested=false 2024-11-21T13:25:27,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:27,281 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:27,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:25:27,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112120acb7f6c5534602823047b07ed2552c_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195526629/Put/seqid=0 2024-11-21T13:25:27,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:27,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742204_1380 (size=12154) 2024-11-21T13:25:27,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195587329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195587329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195587330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195587331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195587332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T13:25:27,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195587450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,454 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195587450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195587450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,455 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195587450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,455 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195587450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,455 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:27,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:27,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:27,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:27,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:27,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:27,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:27,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:27,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195587659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195587659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,667 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195587659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195587659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195587659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,669 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,714 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:27,719 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112120acb7f6c5534602823047b07ed2552c_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112120acb7f6c5534602823047b07ed2552c_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:27,719 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/7d793a93f9be4019a1f82480113c8c75, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:27,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/7d793a93f9be4019a1f82480113c8c75 is 175, key is test_row_0/A:col10/1732195526629/Put/seqid=0 2024-11-21T13:25:27,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742205_1381 (size=30955) 2024-11-21T13:25:27,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:27,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:27,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:27,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:27,770 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:27,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:27,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:27,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:27,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:27,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195587979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195587980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,984 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195587980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,984 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195587980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:27,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195587980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:27,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:27,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:28,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:28,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:28,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:28,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:28,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:28,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:28,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:28,132 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T13:25:28,137 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/7d793a93f9be4019a1f82480113c8c75 2024-11-21T13:25:28,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/b5b4b625e62f4c9da8e3adcdbc81b2b0 is 50, key is test_row_0/B:col10/1732195526629/Put/seqid=0 2024-11-21T13:25:28,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742206_1382 (size=12001) 2024-11-21T13:25:28,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/b5b4b625e62f4c9da8e3adcdbc81b2b0 2024-11-21T13:25:28,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/f12f946a320045818cfcfc4745badef2 is 50, key is test_row_0/C:col10/1732195526629/Put/seqid=0 2024-11-21T13:25:28,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742207_1383 (size=12001) 2024-11-21T13:25:28,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:28,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:28,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:28,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:28,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:28,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:28,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T13:25:28,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:28,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:28,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:28,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:28,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:28,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:28,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:28,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195588489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:28,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:28,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195588490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:28,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:28,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:28,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195588490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:28,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:28,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:28,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195588490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:28,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:28,496 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:28,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195588490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:28,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:28,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:28,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:28,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:28,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:28,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:28,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:28,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:28,574 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/f12f946a320045818cfcfc4745badef2 2024-11-21T13:25:28,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/7d793a93f9be4019a1f82480113c8c75 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/7d793a93f9be4019a1f82480113c8c75 2024-11-21T13:25:28,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/7d793a93f9be4019a1f82480113c8c75, entries=150, sequenceid=55, filesize=30.2 K 2024-11-21T13:25:28,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/b5b4b625e62f4c9da8e3adcdbc81b2b0 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/b5b4b625e62f4c9da8e3adcdbc81b2b0 2024-11-21T13:25:28,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/b5b4b625e62f4c9da8e3adcdbc81b2b0, entries=150, sequenceid=55, filesize=11.7 K 2024-11-21T13:25:28,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/f12f946a320045818cfcfc4745badef2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/f12f946a320045818cfcfc4745badef2 2024-11-21T13:25:28,589 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/f12f946a320045818cfcfc4745badef2, entries=150, sequenceid=55, filesize=11.7 K 2024-11-21T13:25:28,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6e6e9ba8386d0e037fdad07b55820eae in 1309ms, sequenceid=55, compaction requested=true 2024-11-21T13:25:28,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:28,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:28,590 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:28,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:28,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:28,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:28,590 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:28,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:28,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:28,591 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:28,591 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/A is initiating minor compaction (all files) 2024-11-21T13:25:28,591 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/A in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:28,591 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:28,591 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/B is initiating minor compaction (all files) 2024-11-21T13:25:28,591 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/B in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:28,591 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8615e316bb964f34ad9fffb17ffb50f4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/f5cf700f1bd44acd856d83c118a9b4cb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/7d793a93f9be4019a1f82480113c8c75] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=99.1 K 2024-11-21T13:25:28,591 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bf31e3c5a5c945959e8a0619bced5cbd, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/f0dde327118f4ac089a53f3604bc27d7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/b5b4b625e62f4c9da8e3adcdbc81b2b0] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=35.2 K 2024-11-21T13:25:28,591 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:28,591 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8615e316bb964f34ad9fffb17ffb50f4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/f5cf700f1bd44acd856d83c118a9b4cb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/7d793a93f9be4019a1f82480113c8c75] 2024-11-21T13:25:28,591 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting bf31e3c5a5c945959e8a0619bced5cbd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732195526187 2024-11-21T13:25:28,591 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8615e316bb964f34ad9fffb17ffb50f4, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732195526187 2024-11-21T13:25:28,591 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f0dde327118f4ac089a53f3604bc27d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732195526229 2024-11-21T13:25:28,592 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5cf700f1bd44acd856d83c118a9b4cb, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732195526229 2024-11-21T13:25:28,592 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b5b4b625e62f4c9da8e3adcdbc81b2b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732195526629 2024-11-21T13:25:28,592 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d793a93f9be4019a1f82480113c8c75, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732195526629 2024-11-21T13:25:28,600 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#B#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:28,600 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/f639681b8f39427dbcef3d6f4781a05c is 50, key is test_row_0/B:col10/1732195526629/Put/seqid=0 2024-11-21T13:25:28,601 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:28,613 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121f2f1e979e03f4853b02069b0cf6cf90f_6e6e9ba8386d0e037fdad07b55820eae store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:28,615 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121f2f1e979e03f4853b02069b0cf6cf90f_6e6e9ba8386d0e037fdad07b55820eae, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:28,615 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f2f1e979e03f4853b02069b0cf6cf90f_6e6e9ba8386d0e037fdad07b55820eae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:28,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742208_1384 (size=12104) 2024-11-21T13:25:28,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742209_1385 (size=4469) 2024-11-21T13:25:28,686 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-21T13:25:28,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:28,686 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T13:25:28,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212c6ec76f5b2a4d97969fd8ef559f217b_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195527329/Put/seqid=0 2024-11-21T13:25:28,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742210_1386 (size=12154) 2024-11-21T13:25:29,022 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#A#compaction#324 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:29,023 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/5f8089c1bf4b4bd385a2e2c7b4d746af is 175, key is test_row_0/A:col10/1732195526629/Put/seqid=0 2024-11-21T13:25:29,027 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/f639681b8f39427dbcef3d6f4781a05c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/f639681b8f39427dbcef3d6f4781a05c 2024-11-21T13:25:29,035 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/B of 6e6e9ba8386d0e037fdad07b55820eae into f639681b8f39427dbcef3d6f4781a05c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:29,035 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:29,035 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/B, priority=13, startTime=1732195528590; duration=0sec 2024-11-21T13:25:29,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742211_1387 (size=31058) 2024-11-21T13:25:29,036 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:29,036 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:B 2024-11-21T13:25:29,036 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:29,038 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:29,038 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/C is initiating minor compaction (all files) 2024-11-21T13:25:29,038 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/C in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:29,038 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/b91c6f738fa0426ea82f3794fb8d0f4a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/7c587dc0479740dc921213737a32e16a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/f12f946a320045818cfcfc4745badef2] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=35.2 K 2024-11-21T13:25:29,039 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b91c6f738fa0426ea82f3794fb8d0f4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732195526187 2024-11-21T13:25:29,039 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c587dc0479740dc921213737a32e16a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=42, earliestPutTs=1732195526229 2024-11-21T13:25:29,040 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f12f946a320045818cfcfc4745badef2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732195526629 2024-11-21T13:25:29,043 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/5f8089c1bf4b4bd385a2e2c7b4d746af as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5f8089c1bf4b4bd385a2e2c7b4d746af 2024-11-21T13:25:29,048 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#C#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:29,049 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/4d53f5db27c64b93ae1fe9036458a988 is 50, key is test_row_0/C:col10/1732195526629/Put/seqid=0 2024-11-21T13:25:29,049 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/A of 6e6e9ba8386d0e037fdad07b55820eae into 5f8089c1bf4b4bd385a2e2c7b4d746af(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:29,049 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:29,050 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/A, priority=13, startTime=1732195528590; duration=0sec 2024-11-21T13:25:29,050 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:29,050 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:A 2024-11-21T13:25:29,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742212_1388 (size=12104) 2024-11-21T13:25:29,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:29,101 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411212c6ec76f5b2a4d97969fd8ef559f217b_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212c6ec76f5b2a4d97969fd8ef559f217b_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:29,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/47e1041ec44949c49b8ad4e5ed5820dc, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:29,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/47e1041ec44949c49b8ad4e5ed5820dc is 175, key is test_row_0/A:col10/1732195527329/Put/seqid=0 2024-11-21T13:25:29,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742213_1389 (size=30955) 2024-11-21T13:25:29,462 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/4d53f5db27c64b93ae1fe9036458a988 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/4d53f5db27c64b93ae1fe9036458a988 2024-11-21T13:25:29,468 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/C of 6e6e9ba8386d0e037fdad07b55820eae into 4d53f5db27c64b93ae1fe9036458a988(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:29,468 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:29,468 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/C, priority=13, startTime=1732195528590; duration=0sec 2024-11-21T13:25:29,468 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:29,468 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:C 2024-11-21T13:25:29,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:29,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:29,506 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/47e1041ec44949c49b8ad4e5ed5820dc 2024-11-21T13:25:29,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/89fae7bb8d2f4e299c41b4b43a7834ed is 50, key is test_row_0/B:col10/1732195527329/Put/seqid=0 2024-11-21T13:25:29,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195589525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742214_1390 (size=12001) 2024-11-21T13:25:29,533 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/89fae7bb8d2f4e299c41b4b43a7834ed 2024-11-21T13:25:29,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195589525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195589526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195589527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195589527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,539 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/9fa0c7c5f4754b969e325520e8a97925 is 50, key is test_row_0/C:col10/1732195527329/Put/seqid=0 2024-11-21T13:25:29,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742215_1391 (size=12001) 2024-11-21T13:25:29,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195589629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195589639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,647 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195589639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195589639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195589649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195589839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195589859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195589859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195589859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,865 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:29,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:29,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195589869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:29,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:29,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,005 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/9fa0c7c5f4754b969e325520e8a97925 2024-11-21T13:25:30,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/47e1041ec44949c49b8ad4e5ed5820dc as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/47e1041ec44949c49b8ad4e5ed5820dc 2024-11-21T13:25:30,013 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/47e1041ec44949c49b8ad4e5ed5820dc, entries=150, sequenceid=78, filesize=30.2 K 2024-11-21T13:25:30,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/89fae7bb8d2f4e299c41b4b43a7834ed as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/89fae7bb8d2f4e299c41b4b43a7834ed 2024-11-21T13:25:30,019 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/89fae7bb8d2f4e299c41b4b43a7834ed, entries=150, sequenceid=78, filesize=11.7 K 2024-11-21T13:25:30,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/9fa0c7c5f4754b969e325520e8a97925 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/9fa0c7c5f4754b969e325520e8a97925 2024-11-21T13:25:30,024 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/9fa0c7c5f4754b969e325520e8a97925, entries=150, sequenceid=78, filesize=11.7 K 2024-11-21T13:25:30,025 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=80.51 KB/82440 for 6e6e9ba8386d0e037fdad07b55820eae in 1339ms, sequenceid=78, compaction requested=false 2024-11-21T13:25:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:30,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-21T13:25:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-21T13:25:30,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-21T13:25:30,028 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8070 sec 2024-11-21T13:25:30,029 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 3.8110 sec 2024-11-21T13:25:30,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:30,152 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:30,153 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T13:25:30,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112108fb758f18014f8ca25dd294267b81c9_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195529515/Put/seqid=0 2024-11-21T13:25:30,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742216_1392 (size=14594) 2024-11-21T13:25:30,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195590201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195590201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195590201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,213 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195590203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195590203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195590319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195590319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195590319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195590319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195590319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-21T13:25:30,389 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:30,389 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:30,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-21T13:25:30,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T13:25:30,391 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:30,392 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:30,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:30,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T13:25:30,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195590529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195590529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,532 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195590529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195590529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,532 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195590529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,544 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T13:25:30,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:30,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:30,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:30,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:30,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:30,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:30,564 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:30,567 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112108fb758f18014f8ca25dd294267b81c9_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112108fb758f18014f8ca25dd294267b81c9_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:30,567 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/9c7cc3f86c9d4dd7a943cd8f5496e035, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:30,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/9c7cc3f86c9d4dd7a943cd8f5496e035 is 175, key is test_row_0/A:col10/1732195529515/Put/seqid=0 2024-11-21T13:25:30,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742217_1393 (size=39549) 2024-11-21T13:25:30,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T13:25:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:30,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:30,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:30,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:30,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T13:25:30,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195590839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195590839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,846 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,846 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195590839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195590839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195590839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:30,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:30,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T13:25:30,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:30,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:30,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:30,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:30,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:30,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:30,980 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/9c7cc3f86c9d4dd7a943cd8f5496e035 2024-11-21T13:25:30,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/2564a56e1b784199aa9b3ab6af067c1f is 50, key is test_row_0/B:col10/1732195529515/Put/seqid=0 2024-11-21T13:25:31,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742218_1394 (size=12001) 2024-11-21T13:25:31,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/2564a56e1b784199aa9b3ab6af067c1f 2024-11-21T13:25:31,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T13:25:31,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:31,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:31,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:31,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:31,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:31,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:31,015 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/234cd13d749341c9a48aab672f0d7b41 is 50, key is test_row_0/C:col10/1732195529515/Put/seqid=0 2024-11-21T13:25:31,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T13:25:31,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742219_1395 (size=12001) 2024-11-21T13:25:31,155 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T13:25:31,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:31,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:31,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:31,156 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:31,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:31,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:31,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T13:25:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:31,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:31,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:31,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:31,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:31,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195591349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:31,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:31,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:31,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195591359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:31,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195591359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:31,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195591359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:31,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:31,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195591359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:31,361 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:31,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:31,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/234cd13d749341c9a48aab672f0d7b41 2024-11-21T13:25:31,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/9c7cc3f86c9d4dd7a943cd8f5496e035 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/9c7cc3f86c9d4dd7a943cd8f5496e035 2024-11-21T13:25:31,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/9c7cc3f86c9d4dd7a943cd8f5496e035, entries=200, sequenceid=97, filesize=38.6 K 2024-11-21T13:25:31,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/2564a56e1b784199aa9b3ab6af067c1f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/2564a56e1b784199aa9b3ab6af067c1f 2024-11-21T13:25:31,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/2564a56e1b784199aa9b3ab6af067c1f, entries=150, sequenceid=97, filesize=11.7 K 2024-11-21T13:25:31,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/234cd13d749341c9a48aab672f0d7b41 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/234cd13d749341c9a48aab672f0d7b41 2024-11-21T13:25:31,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/234cd13d749341c9a48aab672f0d7b41, entries=150, sequenceid=97, filesize=11.7 K 2024-11-21T13:25:31,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 6e6e9ba8386d0e037fdad07b55820eae in 1283ms, sequenceid=97, compaction requested=true 2024-11-21T13:25:31,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:31,435 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:31,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:31,436 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:31,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:31,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:31,436 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:31,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:31,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:31,436 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:31,436 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:31,436 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/B is initiating minor compaction (all files) 2024-11-21T13:25:31,436 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/A is initiating minor compaction (all files) 2024-11-21T13:25:31,436 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/B in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:31,436 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/A in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:31,437 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/f639681b8f39427dbcef3d6f4781a05c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/89fae7bb8d2f4e299c41b4b43a7834ed, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/2564a56e1b784199aa9b3ab6af067c1f] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=35.3 K 2024-11-21T13:25:31,437 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5f8089c1bf4b4bd385a2e2c7b4d746af, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/47e1041ec44949c49b8ad4e5ed5820dc, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/9c7cc3f86c9d4dd7a943cd8f5496e035] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=99.2 K 2024-11-21T13:25:31,437 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:31,437 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5f8089c1bf4b4bd385a2e2c7b4d746af, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/47e1041ec44949c49b8ad4e5ed5820dc, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/9c7cc3f86c9d4dd7a943cd8f5496e035] 2024-11-21T13:25:31,437 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f639681b8f39427dbcef3d6f4781a05c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732195526629 2024-11-21T13:25:31,437 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f8089c1bf4b4bd385a2e2c7b4d746af, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732195526629 2024-11-21T13:25:31,437 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 89fae7bb8d2f4e299c41b4b43a7834ed, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732195527311 2024-11-21T13:25:31,437 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 47e1041ec44949c49b8ad4e5ed5820dc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732195527311 2024-11-21T13:25:31,437 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 2564a56e1b784199aa9b3ab6af067c1f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732195529515 2024-11-21T13:25:31,437 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9c7cc3f86c9d4dd7a943cd8f5496e035, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732195529515 2024-11-21T13:25:31,442 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:31,444 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#B#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:31,444 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/c8de8ab2a1cc4ef9b36d623ba96d1505 is 50, key is test_row_0/B:col10/1732195529515/Put/seqid=0 2024-11-21T13:25:31,451 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121b2a6d22a8e7045c58a4cc261292ae868_6e6e9ba8386d0e037fdad07b55820eae store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:31,453 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121b2a6d22a8e7045c58a4cc261292ae868_6e6e9ba8386d0e037fdad07b55820eae, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:31,454 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b2a6d22a8e7045c58a4cc261292ae868_6e6e9ba8386d0e037fdad07b55820eae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:31,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742220_1396 (size=12207) 2024-11-21T13:25:31,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-21T13:25:31,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:31,461 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T13:25:31,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742221_1397 (size=4469) 2024-11-21T13:25:31,463 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#A#compaction#332 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:31,464 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/0e82f7082f4444958301b449a5ec433b is 175, key is test_row_0/A:col10/1732195529515/Put/seqid=0 2024-11-21T13:25:31,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121532c99728325425b915918b318a3b14c_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195530189/Put/seqid=0 2024-11-21T13:25:31,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742222_1398 (size=31161) 2024-11-21T13:25:31,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742223_1399 (size=12154) 2024-11-21T13:25:31,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T13:25:31,861 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/c8de8ab2a1cc4ef9b36d623ba96d1505 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/c8de8ab2a1cc4ef9b36d623ba96d1505 2024-11-21T13:25:31,866 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/B of 6e6e9ba8386d0e037fdad07b55820eae into c8de8ab2a1cc4ef9b36d623ba96d1505(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:31,866 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:31,866 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/B, priority=13, startTime=1732195531436; duration=0sec 2024-11-21T13:25:31,866 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:31,866 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:B 2024-11-21T13:25:31,866 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:31,867 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:31,867 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/C is initiating minor compaction (all files) 2024-11-21T13:25:31,867 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/C in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:31,867 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/4d53f5db27c64b93ae1fe9036458a988, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/9fa0c7c5f4754b969e325520e8a97925, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/234cd13d749341c9a48aab672f0d7b41] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=35.3 K 2024-11-21T13:25:31,869 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d53f5db27c64b93ae1fe9036458a988, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1732195526629 2024-11-21T13:25:31,869 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fa0c7c5f4754b969e325520e8a97925, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732195527311 2024-11-21T13:25:31,870 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 234cd13d749341c9a48aab672f0d7b41, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732195529515 2024-11-21T13:25:31,881 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#C#compaction#335 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:31,881 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/7c95d4f7620b40518dd9496b0955a50a is 50, key is test_row_0/C:col10/1732195529515/Put/seqid=0 2024-11-21T13:25:31,882 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/0e82f7082f4444958301b449a5ec433b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0e82f7082f4444958301b449a5ec433b 2024-11-21T13:25:31,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:31,892 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121532c99728325425b915918b318a3b14c_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121532c99728325425b915918b318a3b14c_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:31,892 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/A of 6e6e9ba8386d0e037fdad07b55820eae into 0e82f7082f4444958301b449a5ec433b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:31,892 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:31,892 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/A, priority=13, startTime=1732195531435; duration=0sec 2024-11-21T13:25:31,892 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:31,892 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:A 2024-11-21T13:25:31,897 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/a7876c7264ff4e63b6ca205af6531040, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:31,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/a7876c7264ff4e63b6ca205af6531040 is 175, key is test_row_0/A:col10/1732195530189/Put/seqid=0 2024-11-21T13:25:31,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742224_1400 (size=12207) 2024-11-21T13:25:31,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742225_1401 (size=30955) 2024-11-21T13:25:31,912 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/a7876c7264ff4e63b6ca205af6531040 2024-11-21T13:25:31,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/012fa91ed5fb46c1872a1fe36ac71c6a is 50, key is test_row_0/B:col10/1732195530189/Put/seqid=0 2024-11-21T13:25:31,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742226_1402 (size=12001) 2024-11-21T13:25:31,937 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/012fa91ed5fb46c1872a1fe36ac71c6a 2024-11-21T13:25:31,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/5d196ab7bcc340d1a4fd80a3db4abd03 is 50, key is test_row_0/C:col10/1732195530189/Put/seqid=0 2024-11-21T13:25:31,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742227_1403 (size=12001) 2024-11-21T13:25:32,312 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/7c95d4f7620b40518dd9496b0955a50a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/7c95d4f7620b40518dd9496b0955a50a 2024-11-21T13:25:32,322 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/C of 6e6e9ba8386d0e037fdad07b55820eae into 7c95d4f7620b40518dd9496b0955a50a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:32,322 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:32,322 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/C, priority=13, startTime=1732195531436; duration=0sec 2024-11-21T13:25:32,322 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:32,322 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:C 2024-11-21T13:25:32,363 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/5d196ab7bcc340d1a4fd80a3db4abd03 2024-11-21T13:25:32,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:32,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:32,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/a7876c7264ff4e63b6ca205af6531040 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a7876c7264ff4e63b6ca205af6531040 2024-11-21T13:25:32,378 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a7876c7264ff4e63b6ca205af6531040, entries=150, sequenceid=118, filesize=30.2 K 2024-11-21T13:25:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/012fa91ed5fb46c1872a1fe36ac71c6a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/012fa91ed5fb46c1872a1fe36ac71c6a 2024-11-21T13:25:32,384 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/012fa91ed5fb46c1872a1fe36ac71c6a, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T13:25:32,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/5d196ab7bcc340d1a4fd80a3db4abd03 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/5d196ab7bcc340d1a4fd80a3db4abd03 2024-11-21T13:25:32,390 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/5d196ab7bcc340d1a4fd80a3db4abd03, entries=150, sequenceid=118, filesize=11.7 K 2024-11-21T13:25:32,391 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=46.96 KB/48090 for 6e6e9ba8386d0e037fdad07b55820eae in 930ms, sequenceid=118, compaction requested=false 2024-11-21T13:25:32,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:32,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:32,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-21T13:25:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-21T13:25:32,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:32,393 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:32,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:25:32,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-21T13:25:32,398 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0010 sec 2024-11-21T13:25:32,401 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.0100 sec 2024-11-21T13:25:32,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121dcda7fcfbf214274a747b6ef4f45ccf1_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195532379/Put/seqid=0 2024-11-21T13:25:32,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742228_1404 (size=12254) 2024-11-21T13:25:32,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195592453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195592455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,468 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195592458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,474 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195592467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195592467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-21T13:25:32,540 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:32,541 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:32,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-21T13:25:32,543 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:32,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T13:25:32,544 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:32,544 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:32,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195592569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195592579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195592579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195592590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195592591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T13:25:32,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T13:25:32,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:32,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:32,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:32,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:32,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:32,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:32,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195592779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,802 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195592799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195592799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195592800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,803 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:32,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195592810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:32,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:32,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T13:25:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:32,851 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:32,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:32,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:32,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T13:25:32,860 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:32,864 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121dcda7fcfbf214274a747b6ef4f45ccf1_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dcda7fcfbf214274a747b6ef4f45ccf1_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:32,869 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/0ecf9b02d93a4b118a7349988d6f02f4, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:32,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/0ecf9b02d93a4b118a7349988d6f02f4 is 175, key is test_row_0/A:col10/1732195532379/Put/seqid=0 2024-11-21T13:25:32,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742229_1405 (size=31055) 2024-11-21T13:25:33,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T13:25:33,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:33,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,108 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195593099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195593110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195593110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,121 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195593110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195593119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T13:25:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T13:25:33,285 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/0ecf9b02d93a4b118a7349988d6f02f4 2024-11-21T13:25:33,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/2d00cd04144a4a7285007ff43a4e03e9 is 50, key is test_row_0/B:col10/1732195532379/Put/seqid=0 2024-11-21T13:25:33,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742230_1406 (size=12101) 2024-11-21T13:25:33,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/2d00cd04144a4a7285007ff43a4e03e9 2024-11-21T13:25:33,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T13:25:33,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:33,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/cd619d938b1244b6b26306086af5db2a is 50, key is test_row_0/C:col10/1732195532379/Put/seqid=0 2024-11-21T13:25:33,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742231_1407 (size=12101) 2024-11-21T13:25:33,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/cd619d938b1244b6b26306086af5db2a 2024-11-21T13:25:33,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/0ecf9b02d93a4b118a7349988d6f02f4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0ecf9b02d93a4b118a7349988d6f02f4 2024-11-21T13:25:33,347 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0ecf9b02d93a4b118a7349988d6f02f4, entries=150, sequenceid=134, filesize=30.3 K 2024-11-21T13:25:33,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/2d00cd04144a4a7285007ff43a4e03e9 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/2d00cd04144a4a7285007ff43a4e03e9 2024-11-21T13:25:33,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/2d00cd04144a4a7285007ff43a4e03e9, entries=150, sequenceid=134, filesize=11.8 K 2024-11-21T13:25:33,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/cd619d938b1244b6b26306086af5db2a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/cd619d938b1244b6b26306086af5db2a 2024-11-21T13:25:33,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/cd619d938b1244b6b26306086af5db2a, entries=150, sequenceid=134, filesize=11.8 K 2024-11-21T13:25:33,365 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 6e6e9ba8386d0e037fdad07b55820eae in 971ms, sequenceid=134, compaction requested=true 2024-11-21T13:25:33,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:33,365 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:33,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:33,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:33,365 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:33,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:33,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:33,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:33,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:33,366 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:33,366 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/B is initiating minor compaction (all files) 2024-11-21T13:25:33,366 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/B in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,367 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/c8de8ab2a1cc4ef9b36d623ba96d1505, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/012fa91ed5fb46c1872a1fe36ac71c6a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/2d00cd04144a4a7285007ff43a4e03e9] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=35.5 K 2024-11-21T13:25:33,367 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:33,367 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/A is initiating minor compaction (all files) 2024-11-21T13:25:33,367 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/A in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,367 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0e82f7082f4444958301b449a5ec433b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a7876c7264ff4e63b6ca205af6531040, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0ecf9b02d93a4b118a7349988d6f02f4] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=91.0 K 2024-11-21T13:25:33,367 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,367 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0e82f7082f4444958301b449a5ec433b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a7876c7264ff4e63b6ca205af6531040, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0ecf9b02d93a4b118a7349988d6f02f4] 2024-11-21T13:25:33,368 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting c8de8ab2a1cc4ef9b36d623ba96d1505, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732195529515 2024-11-21T13:25:33,368 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e82f7082f4444958301b449a5ec433b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732195529515 2024-11-21T13:25:33,369 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 012fa91ed5fb46c1872a1fe36ac71c6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732195530185 2024-11-21T13:25:33,369 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7876c7264ff4e63b6ca205af6531040, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732195530185 2024-11-21T13:25:33,369 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d00cd04144a4a7285007ff43a4e03e9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732195532379 2024-11-21T13:25:33,369 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ecf9b02d93a4b118a7349988d6f02f4, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732195532379 2024-11-21T13:25:33,381 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:33,394 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#B#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:33,394 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/bc670dfabd8546fdb6643de98f074a14 is 50, key is test_row_0/B:col10/1732195532379/Put/seqid=0 2024-11-21T13:25:33,395 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121b5d30157aec245a5b23fc4537ecb9cda_6e6e9ba8386d0e037fdad07b55820eae store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:33,397 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121b5d30157aec245a5b23fc4537ecb9cda_6e6e9ba8386d0e037fdad07b55820eae, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:33,398 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b5d30157aec245a5b23fc4537ecb9cda_6e6e9ba8386d0e037fdad07b55820eae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:33,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742232_1408 (size=12409) 2024-11-21T13:25:33,433 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/bc670dfabd8546fdb6643de98f074a14 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bc670dfabd8546fdb6643de98f074a14 2024-11-21T13:25:33,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742233_1409 (size=4469) 2024-11-21T13:25:33,441 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#A#compaction#341 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:33,441 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/e8aeaea3766c4bffbbdf83683b6cc204 is 175, key is test_row_0/A:col10/1732195532379/Put/seqid=0 2024-11-21T13:25:33,442 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/B of 6e6e9ba8386d0e037fdad07b55820eae into bc670dfabd8546fdb6643de98f074a14(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:33,442 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:33,442 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/B, priority=13, startTime=1732195533365; duration=0sec 2024-11-21T13:25:33,442 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:33,442 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:B 2024-11-21T13:25:33,442 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:33,443 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:33,443 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/C is initiating minor compaction (all files) 2024-11-21T13:25:33,444 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/C in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,444 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/7c95d4f7620b40518dd9496b0955a50a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/5d196ab7bcc340d1a4fd80a3db4abd03, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/cd619d938b1244b6b26306086af5db2a] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=35.5 K 2024-11-21T13:25:33,444 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c95d4f7620b40518dd9496b0955a50a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732195529515 2024-11-21T13:25:33,444 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d196ab7bcc340d1a4fd80a3db4abd03, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732195530185 2024-11-21T13:25:33,445 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting cd619d938b1244b6b26306086af5db2a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732195532379 2024-11-21T13:25:33,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742234_1410 (size=31363) 2024-11-21T13:25:33,458 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#C#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:33,459 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/3555acc83d5c41fdb305d4ce30476a00 is 50, key is test_row_0/C:col10/1732195532379/Put/seqid=0 2024-11-21T13:25:33,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-21T13:25:33,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,467 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T13:25:33,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742235_1411 (size=12409) 2024-11-21T13:25:33,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213539ef12110b4064b028be66bec661a1_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195532435/Put/seqid=0 2024-11-21T13:25:33,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742236_1412 (size=12304) 2024-11-21T13:25:33,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,545 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411213539ef12110b4064b028be66bec661a1_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213539ef12110b4064b028be66bec661a1_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:33,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/d47c11254f2644e89b4037f0d2914b34, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:33,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/d47c11254f2644e89b4037f0d2914b34 is 175, key is test_row_0/A:col10/1732195532435/Put/seqid=0 2024-11-21T13:25:33,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742237_1413 (size=31105) 2024-11-21T13:25:33,560 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=159, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/d47c11254f2644e89b4037f0d2914b34 2024-11-21T13:25:33,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/13901c1ebd004518bd35b3ecc98522bb is 50, key is test_row_0/B:col10/1732195532435/Put/seqid=0 2024-11-21T13:25:33,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742238_1414 (size=12151) 2024-11-21T13:25:33,587 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/13901c1ebd004518bd35b3ecc98522bb 2024-11-21T13:25:33,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/e6e8e66c9a3a43518751d49847bcabc4 is 50, key is test_row_0/C:col10/1732195532435/Put/seqid=0 2024-11-21T13:25:33,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742239_1415 (size=12151) 2024-11-21T13:25:33,626 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=159 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/e6e8e66c9a3a43518751d49847bcabc4 2024-11-21T13:25:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:33,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:33,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/d47c11254f2644e89b4037f0d2914b34 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/d47c11254f2644e89b4037f0d2914b34 2024-11-21T13:25:33,638 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/d47c11254f2644e89b4037f0d2914b34, entries=150, sequenceid=159, filesize=30.4 K 2024-11-21T13:25:33,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/13901c1ebd004518bd35b3ecc98522bb as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/13901c1ebd004518bd35b3ecc98522bb 2024-11-21T13:25:33,644 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/13901c1ebd004518bd35b3ecc98522bb, entries=150, sequenceid=159, filesize=11.9 K 2024-11-21T13:25:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/e6e8e66c9a3a43518751d49847bcabc4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/e6e8e66c9a3a43518751d49847bcabc4 2024-11-21T13:25:33,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,648 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/e6e8e66c9a3a43518751d49847bcabc4, entries=150, sequenceid=159, filesize=11.9 K 2024-11-21T13:25:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,649 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=40.25 KB/41220 for 6e6e9ba8386d0e037fdad07b55820eae in 182ms, sequenceid=159, compaction requested=false 2024-11-21T13:25:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:33,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-21T13:25:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-21T13:25:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-21T13:25:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1070 sec 2024-11-21T13:25:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.1130 sec 2024-11-21T13:25:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-21T13:25:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,680 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:33,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,681 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-21T13:25:33,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T13:25:33,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:33,686 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:33,686 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:25:33,687 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:33,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,689 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:33,689 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:33,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219affe5dcade84fb7aeadd49ffeeb1185_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195533682/Put/seqid=0 2024-11-21T13:25:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742240_1416 (size=19774) 2024-11-21T13:25:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:33,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195593773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195593775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T13:25:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195593782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195593783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195593789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T13:25:33,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:33,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,862 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/e8aeaea3766c4bffbbdf83683b6cc204 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/e8aeaea3766c4bffbbdf83683b6cc204 2024-11-21T13:25:33,867 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/A of 6e6e9ba8386d0e037fdad07b55820eae into e8aeaea3766c4bffbbdf83683b6cc204(size=30.6 K), total size for store is 61.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:33,867 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:33,867 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/A, priority=13, startTime=1732195533365; duration=0sec 2024-11-21T13:25:33,867 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:33,868 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:A 2024-11-21T13:25:33,904 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/3555acc83d5c41fdb305d4ce30476a00 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/3555acc83d5c41fdb305d4ce30476a00 2024-11-21T13:25:33,911 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/C of 6e6e9ba8386d0e037fdad07b55820eae into 3555acc83d5c41fdb305d4ce30476a00(size=12.1 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:33,911 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:33,911 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/C, priority=13, startTime=1732195533365; duration=0sec 2024-11-21T13:25:33,911 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:33,911 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:C 2024-11-21T13:25:33,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195593899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195593899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195593899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195593899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,917 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:33,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195593909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:33,918 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:33,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T13:25:33,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:33,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:33,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T13:25:34,113 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:34,116 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219affe5dcade84fb7aeadd49ffeeb1185_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219affe5dcade84fb7aeadd49ffeeb1185_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:34,117 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/35c2801afeb040b3ad9fee066718a373, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:34,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/35c2801afeb040b3ad9fee066718a373 is 175, key is test_row_0/A:col10/1732195533682/Put/seqid=0 2024-11-21T13:25:34,120 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195594119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195594119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742241_1417 (size=57033) 2024-11-21T13:25:34,124 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/35c2801afeb040b3ad9fee066718a373 2024-11-21T13:25:34,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/344cb3bb052c4a7c8cc7caa6e80f34eb is 50, key is test_row_0/B:col10/1732195533682/Put/seqid=0 2024-11-21T13:25:34,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195594129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195594129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195594129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742242_1418 (size=12151) 2024-11-21T13:25:34,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T13:25:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,308 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T13:25:34,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:34,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T13:25:34,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195594430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195594430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,442 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,443 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195594439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195594439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,446 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195594439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,447 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,463 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T13:25:34,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:34,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,464 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/344cb3bb052c4a7c8cc7caa6e80f34eb 2024-11-21T13:25:34,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/45433c3d724a4adcaf1a5aa66b335f1a is 50, key is test_row_0/C:col10/1732195533682/Put/seqid=0 2024-11-21T13:25:34,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742243_1419 (size=12151) 2024-11-21T13:25:34,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T13:25:34,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:34,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T13:25:34,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:34,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T13:25:34,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T13:25:34,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,926 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:34,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195594949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195594949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195594949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195594959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195594959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:34,966 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:34,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/45433c3d724a4adcaf1a5aa66b335f1a 2024-11-21T13:25:34,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/35c2801afeb040b3ad9fee066718a373 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/35c2801afeb040b3ad9fee066718a373 2024-11-21T13:25:34,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/35c2801afeb040b3ad9fee066718a373, entries=300, sequenceid=172, filesize=55.7 K 2024-11-21T13:25:34,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/344cb3bb052c4a7c8cc7caa6e80f34eb as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/344cb3bb052c4a7c8cc7caa6e80f34eb 2024-11-21T13:25:34,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/344cb3bb052c4a7c8cc7caa6e80f34eb, entries=150, sequenceid=172, filesize=11.9 K 2024-11-21T13:25:34,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/45433c3d724a4adcaf1a5aa66b335f1a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/45433c3d724a4adcaf1a5aa66b335f1a 2024-11-21T13:25:34,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/45433c3d724a4adcaf1a5aa66b335f1a, entries=150, sequenceid=172, filesize=11.9 K 2024-11-21T13:25:34,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 6e6e9ba8386d0e037fdad07b55820eae in 1304ms, sequenceid=172, compaction requested=true 2024-11-21T13:25:34,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:34,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:34,990 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:34,990 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:34,990 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:34,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:34,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:34,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:34,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:34,992 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119501 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:34,992 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:34,992 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/B is initiating minor compaction (all files) 2024-11-21T13:25:34,992 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/A is initiating minor compaction (all files) 2024-11-21T13:25:34,992 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/B in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,992 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/A in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,992 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/e8aeaea3766c4bffbbdf83683b6cc204, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/d47c11254f2644e89b4037f0d2914b34, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/35c2801afeb040b3ad9fee066718a373] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=116.7 K 2024-11-21T13:25:34,992 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bc670dfabd8546fdb6643de98f074a14, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/13901c1ebd004518bd35b3ecc98522bb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/344cb3bb052c4a7c8cc7caa6e80f34eb] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=35.9 K 2024-11-21T13:25:34,992 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:34,992 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/e8aeaea3766c4bffbbdf83683b6cc204, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/d47c11254f2644e89b4037f0d2914b34, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/35c2801afeb040b3ad9fee066718a373] 2024-11-21T13:25:34,992 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting bc670dfabd8546fdb6643de98f074a14, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732195532379 2024-11-21T13:25:34,992 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8aeaea3766c4bffbbdf83683b6cc204, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732195532379 2024-11-21T13:25:34,993 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 13901c1ebd004518bd35b3ecc98522bb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732195532435 2024-11-21T13:25:34,993 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting d47c11254f2644e89b4037f0d2914b34, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732195532435 2024-11-21T13:25:34,993 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 344cb3bb052c4a7c8cc7caa6e80f34eb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732195533682 2024-11-21T13:25:34,993 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35c2801afeb040b3ad9fee066718a373, keycount=300, bloomtype=ROW, size=55.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732195533633 2024-11-21T13:25:35,001 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:35,002 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#B#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:35,002 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/b1ad0abb31a445bcad45689c92a9ef1e is 50, key is test_row_0/B:col10/1732195533682/Put/seqid=0 2024-11-21T13:25:35,003 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121a41b0193283f4fae9da363eeff9a4605_6e6e9ba8386d0e037fdad07b55820eae store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:35,007 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121a41b0193283f4fae9da363eeff9a4605_6e6e9ba8386d0e037fdad07b55820eae, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:35,008 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a41b0193283f4fae9da363eeff9a4605_6e6e9ba8386d0e037fdad07b55820eae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:35,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742244_1420 (size=12561) 2024-11-21T13:25:35,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742245_1421 (size=4469) 2024-11-21T13:25:35,023 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/b1ad0abb31a445bcad45689c92a9ef1e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/b1ad0abb31a445bcad45689c92a9ef1e 2024-11-21T13:25:35,023 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#A#compaction#351 average throughput is 1.16 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:35,023 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/471cd01c258147688205a5d6a8945517 is 175, key is test_row_0/A:col10/1732195533682/Put/seqid=0 2024-11-21T13:25:35,027 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/B of 6e6e9ba8386d0e037fdad07b55820eae into b1ad0abb31a445bcad45689c92a9ef1e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:35,027 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:35,027 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/B, priority=13, startTime=1732195534990; duration=0sec 2024-11-21T13:25:35,027 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:35,027 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:B 2024-11-21T13:25:35,027 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:35,028 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:35,028 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/C is initiating minor compaction (all files) 2024-11-21T13:25:35,028 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/C in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:35,028 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/3555acc83d5c41fdb305d4ce30476a00, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/e6e8e66c9a3a43518751d49847bcabc4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/45433c3d724a4adcaf1a5aa66b335f1a] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=35.9 K 2024-11-21T13:25:35,029 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 3555acc83d5c41fdb305d4ce30476a00, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732195532379 2024-11-21T13:25:35,029 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e6e8e66c9a3a43518751d49847bcabc4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=159, earliestPutTs=1732195532435 2024-11-21T13:25:35,029 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 45433c3d724a4adcaf1a5aa66b335f1a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732195533682 2024-11-21T13:25:35,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742246_1422 (size=31515) 2024-11-21T13:25:35,038 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#C#compaction#352 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:35,038 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/6c05a7b123044d0dab92b78bbfe4ba58 is 50, key is test_row_0/C:col10/1732195533682/Put/seqid=0 2024-11-21T13:25:35,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742247_1423 (size=12561) 2024-11-21T13:25:35,075 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/6c05a7b123044d0dab92b78bbfe4ba58 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/6c05a7b123044d0dab92b78bbfe4ba58 2024-11-21T13:25:35,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-21T13:25:35,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:35,078 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T13:25:35,087 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/C of 6e6e9ba8386d0e037fdad07b55820eae into 6c05a7b123044d0dab92b78bbfe4ba58(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:35,088 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:35,088 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/C, priority=13, startTime=1732195534991; duration=0sec 2024-11-21T13:25:35,088 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:35,088 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:C 2024-11-21T13:25:35,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121857cb7d05f314c87bda98e8d4b0f033f_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195533773/Put/seqid=0 2024-11-21T13:25:35,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742248_1424 (size=12304) 2024-11-21T13:25:35,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:35,100 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121857cb7d05f314c87bda98e8d4b0f033f_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121857cb7d05f314c87bda98e8d4b0f033f_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:35,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/2de5f65aba834691adf1eff5f52c2397, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:35,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/2de5f65aba834691adf1eff5f52c2397 is 175, key is test_row_0/A:col10/1732195533773/Put/seqid=0 2024-11-21T13:25:35,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742249_1425 (size=31105) 2024-11-21T13:25:35,437 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/471cd01c258147688205a5d6a8945517 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/471cd01c258147688205a5d6a8945517 2024-11-21T13:25:35,443 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/A of 6e6e9ba8386d0e037fdad07b55820eae into 471cd01c258147688205a5d6a8945517(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:35,443 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:35,443 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/A, priority=13, startTime=1732195534990; duration=0sec 2024-11-21T13:25:35,443 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:35,443 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:A 2024-11-21T13:25:35,520 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/2de5f65aba834691adf1eff5f52c2397 2024-11-21T13:25:35,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/59aadb2dd3734ed29953561270e67f25 is 50, key is test_row_0/B:col10/1732195533773/Put/seqid=0 2024-11-21T13:25:35,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742250_1426 (size=12151) 2024-11-21T13:25:35,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T13:25:35,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:35,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:35,985 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/59aadb2dd3734ed29953561270e67f25 2024-11-21T13:25:36,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/db9f135a1e3a4e4a8e03ffc8f11eb377 is 50, key is test_row_0/C:col10/1732195533773/Put/seqid=0 2024-11-21T13:25:36,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195595984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195595990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195596014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,026 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195596012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195596014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742251_1427 (size=12151) 2024-11-21T13:25:36,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195596119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195596120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195596129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195596139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195596139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,346 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195596339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195596340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195596349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195596349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195596349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,475 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/db9f135a1e3a4e4a8e03ffc8f11eb377 2024-11-21T13:25:36,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/2de5f65aba834691adf1eff5f52c2397 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/2de5f65aba834691adf1eff5f52c2397 2024-11-21T13:25:36,487 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/2de5f65aba834691adf1eff5f52c2397, entries=150, sequenceid=198, filesize=30.4 K 2024-11-21T13:25:36,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/59aadb2dd3734ed29953561270e67f25 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/59aadb2dd3734ed29953561270e67f25 2024-11-21T13:25:36,502 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/59aadb2dd3734ed29953561270e67f25, entries=150, sequenceid=198, filesize=11.9 K 2024-11-21T13:25:36,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/db9f135a1e3a4e4a8e03ffc8f11eb377 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/db9f135a1e3a4e4a8e03ffc8f11eb377 2024-11-21T13:25:36,507 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/db9f135a1e3a4e4a8e03ffc8f11eb377, entries=150, sequenceid=198, filesize=11.9 K 2024-11-21T13:25:36,510 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 6e6e9ba8386d0e037fdad07b55820eae in 1431ms, sequenceid=198, compaction requested=false 2024-11-21T13:25:36,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:36,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:36,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-21T13:25:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-21T13:25:36,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-21T13:25:36,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8220 sec 2024-11-21T13:25:36,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.8350 sec 2024-11-21T13:25:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:36,665 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:36,666 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T13:25:36,693 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217f71d7cf93c9494cb63377ccc3d0416e_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195536664/Put/seqid=0 2024-11-21T13:25:36,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195596723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742252_1428 (size=17284) 2024-11-21T13:25:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195596728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195596726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,748 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195596730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195596732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195596849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,860 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195596859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195596860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195596860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195596860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:36,873 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,076 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195597069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,076 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195597080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195597080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195597080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195597080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,149 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:37,163 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217f71d7cf93c9494cb63377ccc3d0416e_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217f71d7cf93c9494cb63377ccc3d0416e_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:37,164 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/8541055b1c124b348083426ccf023670, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:37,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/8541055b1c124b348083426ccf023670 is 175, key is test_row_0/A:col10/1732195536664/Put/seqid=0 2024-11-21T13:25:37,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742253_1429 (size=48389) 2024-11-21T13:25:37,209 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=216, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/8541055b1c124b348083426ccf023670 2024-11-21T13:25:37,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/9c7f52ca2fb54d13bc61e492517f9134 is 50, key is test_row_0/B:col10/1732195536664/Put/seqid=0 2024-11-21T13:25:37,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742254_1430 (size=12151) 2024-11-21T13:25:37,246 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/9c7f52ca2fb54d13bc61e492517f9134 2024-11-21T13:25:37,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/97e4a12049cf4f3fb08029785ef1bf99 is 50, key is test_row_0/C:col10/1732195536664/Put/seqid=0 2024-11-21T13:25:37,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742255_1431 (size=12151) 2024-11-21T13:25:37,262 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=216 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/97e4a12049cf4f3fb08029785ef1bf99 2024-11-21T13:25:37,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/8541055b1c124b348083426ccf023670 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8541055b1c124b348083426ccf023670 2024-11-21T13:25:37,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8541055b1c124b348083426ccf023670, entries=250, sequenceid=216, filesize=47.3 K 2024-11-21T13:25:37,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/9c7f52ca2fb54d13bc61e492517f9134 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/9c7f52ca2fb54d13bc61e492517f9134 2024-11-21T13:25:37,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/9c7f52ca2fb54d13bc61e492517f9134, entries=150, sequenceid=216, filesize=11.9 K 2024-11-21T13:25:37,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/97e4a12049cf4f3fb08029785ef1bf99 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/97e4a12049cf4f3fb08029785ef1bf99 2024-11-21T13:25:37,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/97e4a12049cf4f3fb08029785ef1bf99, entries=150, sequenceid=216, filesize=11.9 K 2024-11-21T13:25:37,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 6e6e9ba8386d0e037fdad07b55820eae in 634ms, sequenceid=216, compaction requested=true 2024-11-21T13:25:37,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:37,300 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:37,306 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111009 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:37,306 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/A is initiating minor compaction (all files) 2024-11-21T13:25:37,306 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/A in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:37,306 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/471cd01c258147688205a5d6a8945517, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/2de5f65aba834691adf1eff5f52c2397, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8541055b1c124b348083426ccf023670] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=108.4 K 2024-11-21T13:25:37,306 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:37,306 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/471cd01c258147688205a5d6a8945517, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/2de5f65aba834691adf1eff5f52c2397, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8541055b1c124b348083426ccf023670] 2024-11-21T13:25:37,307 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 471cd01c258147688205a5d6a8945517, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732195533682 2024-11-21T13:25:37,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:37,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:37,307 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2de5f65aba834691adf1eff5f52c2397, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732195533773 2024-11-21T13:25:37,307 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:37,307 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8541055b1c124b348083426ccf023670, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732195535978 2024-11-21T13:25:37,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:37,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:37,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:37,308 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:37,308 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:37,308 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/B is initiating minor compaction (all files) 2024-11-21T13:25:37,308 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/B in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:37,308 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/b1ad0abb31a445bcad45689c92a9ef1e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/59aadb2dd3734ed29953561270e67f25, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/9c7f52ca2fb54d13bc61e492517f9134] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=36.0 K 2024-11-21T13:25:37,308 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b1ad0abb31a445bcad45689c92a9ef1e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732195533682 2024-11-21T13:25:37,312 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 59aadb2dd3734ed29953561270e67f25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732195533773 2024-11-21T13:25:37,313 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c7f52ca2fb54d13bc61e492517f9134, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732195536011 2024-11-21T13:25:37,318 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:37,323 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411219018203505724d6c8cef428724359a1c_6e6e9ba8386d0e037fdad07b55820eae store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:37,325 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411219018203505724d6c8cef428724359a1c_6e6e9ba8386d0e037fdad07b55820eae, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:37,325 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219018203505724d6c8cef428724359a1c_6e6e9ba8386d0e037fdad07b55820eae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:37,342 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:37,343 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/d7c7636764d347e48cd002a79ed34ec7 is 50, key is test_row_0/B:col10/1732195536664/Put/seqid=0 2024-11-21T13:25:37,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742256_1432 (size=4469) 2024-11-21T13:25:37,374 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#A#compaction#359 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:37,374 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/5e1d757e95a047dea6d06f6bf914af3a is 175, key is test_row_0/A:col10/1732195536664/Put/seqid=0 2024-11-21T13:25:37,386 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:37,386 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T13:25:37,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742257_1433 (size=12663) 2024-11-21T13:25:37,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742258_1434 (size=31617) 2024-11-21T13:25:37,408 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/5e1d757e95a047dea6d06f6bf914af3a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5e1d757e95a047dea6d06f6bf914af3a 2024-11-21T13:25:37,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a41ab0450fd94a449c058df8ad860205_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195536727/Put/seqid=0 2024-11-21T13:25:37,421 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/A of 6e6e9ba8386d0e037fdad07b55820eae into 5e1d757e95a047dea6d06f6bf914af3a(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:37,421 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:37,421 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/A, priority=13, startTime=1732195537300; duration=0sec 2024-11-21T13:25:37,421 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:37,421 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:A 2024-11-21T13:25:37,421 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:37,422 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:37,422 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/C is initiating minor compaction (all files) 2024-11-21T13:25:37,422 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/C in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:37,422 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/6c05a7b123044d0dab92b78bbfe4ba58, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/db9f135a1e3a4e4a8e03ffc8f11eb377, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/97e4a12049cf4f3fb08029785ef1bf99] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=36.0 K 2024-11-21T13:25:37,422 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c05a7b123044d0dab92b78bbfe4ba58, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732195533682 2024-11-21T13:25:37,423 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting db9f135a1e3a4e4a8e03ffc8f11eb377, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1732195533773 2024-11-21T13:25:37,423 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97e4a12049cf4f3fb08029785ef1bf99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732195536011 2024-11-21T13:25:37,443 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#C#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:37,443 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/e7c6102d3e054038bdfcd29446e2edb4 is 50, key is test_row_0/C:col10/1732195536664/Put/seqid=0 2024-11-21T13:25:37,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742259_1435 (size=14794) 2024-11-21T13:25:37,447 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:37,451 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a41ab0450fd94a449c058df8ad860205_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a41ab0450fd94a449c058df8ad860205_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:37,452 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/464ed647de6b493780d3d765be0c0523, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:37,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/464ed647de6b493780d3d765be0c0523 is 175, key is test_row_0/A:col10/1732195536727/Put/seqid=0 2024-11-21T13:25:37,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195597444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,465 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195597449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195597449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195597451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195597448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742260_1436 (size=12663) 2024-11-21T13:25:37,479 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/e7c6102d3e054038bdfcd29446e2edb4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/e7c6102d3e054038bdfcd29446e2edb4 2024-11-21T13:25:37,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742261_1437 (size=39749) 2024-11-21T13:25:37,489 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/464ed647de6b493780d3d765be0c0523 2024-11-21T13:25:37,491 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/C of 6e6e9ba8386d0e037fdad07b55820eae into e7c6102d3e054038bdfcd29446e2edb4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:37,491 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:37,491 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/C, priority=13, startTime=1732195537308; duration=0sec 2024-11-21T13:25:37,491 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:37,491 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:C 2024-11-21T13:25:37,521 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/8efbd12138584821ba6a76f6179202cc is 50, key is test_row_0/B:col10/1732195536727/Put/seqid=0 2024-11-21T13:25:37,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742262_1438 (size=12151) 2024-11-21T13:25:37,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/8efbd12138584821ba6a76f6179202cc 2024-11-21T13:25:37,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/a9b58972c2a84b1d95db3c65a5c8d849 is 50, key is test_row_0/C:col10/1732195536727/Put/seqid=0 2024-11-21T13:25:37,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195597569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195597569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195597569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,577 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195597569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,578 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195597569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742263_1439 (size=12151) 2024-11-21T13:25:37,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/a9b58972c2a84b1d95db3c65a5c8d849 2024-11-21T13:25:37,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/464ed647de6b493780d3d765be0c0523 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/464ed647de6b493780d3d765be0c0523 2024-11-21T13:25:37,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/464ed647de6b493780d3d765be0c0523, entries=200, sequenceid=237, filesize=38.8 K 2024-11-21T13:25:37,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/8efbd12138584821ba6a76f6179202cc as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/8efbd12138584821ba6a76f6179202cc 2024-11-21T13:25:37,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/8efbd12138584821ba6a76f6179202cc, entries=150, sequenceid=237, filesize=11.9 K 2024-11-21T13:25:37,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/a9b58972c2a84b1d95db3c65a5c8d849 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/a9b58972c2a84b1d95db3c65a5c8d849 2024-11-21T13:25:37,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/a9b58972c2a84b1d95db3c65a5c8d849, entries=150, sequenceid=237, filesize=11.9 K 2024-11-21T13:25:37,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6e6e9ba8386d0e037fdad07b55820eae in 247ms, sequenceid=237, compaction requested=false 2024-11-21T13:25:37,633 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:37,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:37,781 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:37,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T13:25:37,788 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112173d5a609d3494fe8968045e07fd4237e_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195537439/Put/seqid=0 2024-11-21T13:25:37,797 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/d7c7636764d347e48cd002a79ed34ec7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/d7c7636764d347e48cd002a79ed34ec7 2024-11-21T13:25:37,803 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/B of 6e6e9ba8386d0e037fdad07b55820eae into d7c7636764d347e48cd002a79ed34ec7(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:37,803 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:37,803 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/B, priority=13, startTime=1732195537307; duration=0sec 2024-11-21T13:25:37,804 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:37,804 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:B 2024-11-21T13:25:37,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742264_1440 (size=14794) 2024-11-21T13:25:37,818 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:37,821 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112173d5a609d3494fe8968045e07fd4237e_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112173d5a609d3494fe8968045e07fd4237e_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:37,823 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/34703934102e4838b6b357e636263939, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:37,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/34703934102e4838b6b357e636263939 is 175, key is test_row_0/A:col10/1732195537439/Put/seqid=0 2024-11-21T13:25:37,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195597823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742265_1441 (size=39749) 2024-11-21T13:25:37,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195597824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,834 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195597824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195597826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195597828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-21T13:25:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,839 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,839 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:37,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:37,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-21T13:25:37,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T13:25:37,844 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:37,844 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:37,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:37,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195597939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195597939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195597939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T13:25:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195597949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195597950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:37,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:37,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T13:25:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:37,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:37,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T13:25:38,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:38,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:38,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:38,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T13:25:38,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:38,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195598159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:38,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195598159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,167 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:38,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:38,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195598163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:38,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:38,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195598165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:38,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195598165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,174 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:38,229 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/34703934102e4838b6b357e636263939 2024-11-21T13:25:38,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/ca12e02939ce415ba3065bc5908e4bea is 50, key is test_row_0/B:col10/1732195537439/Put/seqid=0 2024-11-21T13:25:38,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742266_1442 (size=12151) 2024-11-21T13:25:38,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T13:25:38,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:38,308 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T13:25:38,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:38,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:38,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:38,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T13:25:38,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:38,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195598470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:38,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:38,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195598479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:38,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195598479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195598479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:38,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195598479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:38,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T13:25:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:38,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/ca12e02939ce415ba3065bc5908e4bea 2024-11-21T13:25:38,678 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/fd11084ceb144cb59bbb14248e3e8de8 is 50, key is test_row_0/C:col10/1732195537439/Put/seqid=0 2024-11-21T13:25:38,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742267_1443 (size=12151) 2024-11-21T13:25:38,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T13:25:38,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:38,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:38,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:38,767 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T13:25:38,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:38,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:38,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:38,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T13:25:38,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:38,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195598989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:38,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195599000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:39,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:39,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:39,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195599000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:39,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:39,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195599000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:39,007 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:39,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195599000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:39,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T13:25:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:39,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:39,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:39,116 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/fd11084ceb144cb59bbb14248e3e8de8 2024-11-21T13:25:39,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/34703934102e4838b6b357e636263939 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/34703934102e4838b6b357e636263939 2024-11-21T13:25:39,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/34703934102e4838b6b357e636263939, entries=200, sequenceid=255, filesize=38.8 K 2024-11-21T13:25:39,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/ca12e02939ce415ba3065bc5908e4bea as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/ca12e02939ce415ba3065bc5908e4bea 2024-11-21T13:25:39,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/ca12e02939ce415ba3065bc5908e4bea, entries=150, sequenceid=255, filesize=11.9 K 2024-11-21T13:25:39,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/fd11084ceb144cb59bbb14248e3e8de8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/fd11084ceb144cb59bbb14248e3e8de8 2024-11-21T13:25:39,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/fd11084ceb144cb59bbb14248e3e8de8, entries=150, sequenceid=255, filesize=11.9 K 2024-11-21T13:25:39,145 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 6e6e9ba8386d0e037fdad07b55820eae in 1364ms, sequenceid=255, compaction requested=true 2024-11-21T13:25:39,145 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:39,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:39,145 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:39,145 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:39,145 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:39,146 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:39,146 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/A is initiating minor compaction (all files) 2024-11-21T13:25:39,146 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/A in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:39,146 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5e1d757e95a047dea6d06f6bf914af3a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/464ed647de6b493780d3d765be0c0523, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/34703934102e4838b6b357e636263939] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=108.5 K 2024-11-21T13:25:39,147 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:39,147 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5e1d757e95a047dea6d06f6bf914af3a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/464ed647de6b493780d3d765be0c0523, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/34703934102e4838b6b357e636263939] 2024-11-21T13:25:39,147 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:39,147 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/B is initiating minor compaction (all files) 2024-11-21T13:25:39,147 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/B in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:39,147 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/d7c7636764d347e48cd002a79ed34ec7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/8efbd12138584821ba6a76f6179202cc, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/ca12e02939ce415ba3065bc5908e4bea] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=36.1 K 2024-11-21T13:25:39,147 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting d7c7636764d347e48cd002a79ed34ec7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732195536011 2024-11-21T13:25:39,147 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e1d757e95a047dea6d06f6bf914af3a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732195536011 2024-11-21T13:25:39,148 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 8efbd12138584821ba6a76f6179202cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732195536693 2024-11-21T13:25:39,148 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 464ed647de6b493780d3d765be0c0523, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732195536693 2024-11-21T13:25:39,148 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting ca12e02939ce415ba3065bc5908e4bea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732195537439 2024-11-21T13:25:39,148 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34703934102e4838b6b357e636263939, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732195537439 2024-11-21T13:25:39,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:39,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:39,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:39,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:39,163 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#B#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:39,164 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/ec384c27587a48c7a1c99effb44a0689 is 50, key is test_row_0/B:col10/1732195537439/Put/seqid=0 2024-11-21T13:25:39,165 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:39,170 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121640a23b628104b359acf5049221e5121_6e6e9ba8386d0e037fdad07b55820eae store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:39,173 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121640a23b628104b359acf5049221e5121_6e6e9ba8386d0e037fdad07b55820eae, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:39,173 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121640a23b628104b359acf5049221e5121_6e6e9ba8386d0e037fdad07b55820eae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:39,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742268_1444 (size=12765) 2024-11-21T13:25:39,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742269_1445 (size=4469) 2024-11-21T13:25:39,196 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#A#compaction#369 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:39,197 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/4193922f88f944dc873d08df7f9cfbcf is 175, key is test_row_0/A:col10/1732195537439/Put/seqid=0 2024-11-21T13:25:39,198 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/ec384c27587a48c7a1c99effb44a0689 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/ec384c27587a48c7a1c99effb44a0689 2024-11-21T13:25:39,207 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/B of 6e6e9ba8386d0e037fdad07b55820eae into ec384c27587a48c7a1c99effb44a0689(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:39,207 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:39,207 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/B, priority=13, startTime=1732195539145; duration=0sec 2024-11-21T13:25:39,207 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:39,207 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:B 2024-11-21T13:25:39,207 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:39,209 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:39,210 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/C is initiating minor compaction (all files) 2024-11-21T13:25:39,210 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/C in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:39,210 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/e7c6102d3e054038bdfcd29446e2edb4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/a9b58972c2a84b1d95db3c65a5c8d849, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/fd11084ceb144cb59bbb14248e3e8de8] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=36.1 K 2024-11-21T13:25:39,210 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e7c6102d3e054038bdfcd29446e2edb4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=216, earliestPutTs=1732195536011 2024-11-21T13:25:39,210 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting a9b58972c2a84b1d95db3c65a5c8d849, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1732195536693 2024-11-21T13:25:39,211 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting fd11084ceb144cb59bbb14248e3e8de8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732195537439 2024-11-21T13:25:39,217 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#C#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:39,217 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/a9731f49a5394c47b85d9f8de12a1578 is 50, key is test_row_0/C:col10/1732195537439/Put/seqid=0 2024-11-21T13:25:39,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742270_1446 (size=31719) 2024-11-21T13:25:39,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742271_1447 (size=12765) 2024-11-21T13:25:39,223 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/4193922f88f944dc873d08df7f9cfbcf as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/4193922f88f944dc873d08df7f9cfbcf 2024-11-21T13:25:39,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-21T13:25:39,228 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/a9731f49a5394c47b85d9f8de12a1578 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/a9731f49a5394c47b85d9f8de12a1578 2024-11-21T13:25:39,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:39,230 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/A of 6e6e9ba8386d0e037fdad07b55820eae into 4193922f88f944dc873d08df7f9cfbcf(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:39,230 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T13:25:39,230 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:39,230 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/A, priority=13, startTime=1732195539145; duration=0sec 2024-11-21T13:25:39,231 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:39,231 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:A 2024-11-21T13:25:39,234 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/C of 6e6e9ba8386d0e037fdad07b55820eae into a9731f49a5394c47b85d9f8de12a1578(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:39,234 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:39,234 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/C, priority=13, startTime=1732195539155; duration=0sec 2024-11-21T13:25:39,234 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:39,234 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:C 2024-11-21T13:25:39,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c15706cc554247a9b774ad4ce8a08c7a_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195537822/Put/seqid=0 2024-11-21T13:25:39,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742272_1448 (size=12454) 2024-11-21T13:25:39,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,246 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121c15706cc554247a9b774ad4ce8a08c7a_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c15706cc554247a9b774ad4ce8a08c7a_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:39,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/8831da03cc874e018ab6e5e750d8a535, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:39,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/8831da03cc874e018ab6e5e750d8a535 is 175, key is test_row_0/A:col10/1732195537822/Put/seqid=0 2024-11-21T13:25:39,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742273_1449 (size=31255) 2024-11-21T13:25:39,653 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/8831da03cc874e018ab6e5e750d8a535 2024-11-21T13:25:39,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/65d2b88a80194d81861363559a40bc25 is 50, key is test_row_0/B:col10/1732195537822/Put/seqid=0 2024-11-21T13:25:39,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742274_1450 (size=12301) 2024-11-21T13:25:39,678 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/65d2b88a80194d81861363559a40bc25 2024-11-21T13:25:39,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/8cc40f607b3647f4b7f2d961e549351e is 50, key is test_row_0/C:col10/1732195537822/Put/seqid=0 2024-11-21T13:25:39,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742275_1451 (size=12301) 2024-11-21T13:25:39,702 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/8cc40f607b3647f4b7f2d961e549351e 2024-11-21T13:25:39,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/8831da03cc874e018ab6e5e750d8a535 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8831da03cc874e018ab6e5e750d8a535 2024-11-21T13:25:39,713 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8831da03cc874e018ab6e5e750d8a535, entries=150, sequenceid=278, filesize=30.5 K 2024-11-21T13:25:39,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/65d2b88a80194d81861363559a40bc25 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/65d2b88a80194d81861363559a40bc25 2024-11-21T13:25:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,721 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/65d2b88a80194d81861363559a40bc25, entries=150, sequenceid=278, filesize=12.0 K 2024-11-21T13:25:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/8cc40f607b3647f4b7f2d961e549351e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/8cc40f607b3647f4b7f2d961e549351e 2024-11-21T13:25:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,726 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/8cc40f607b3647f4b7f2d961e549351e, entries=150, sequenceid=278, filesize=12.0 K 2024-11-21T13:25:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,727 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 6e6e9ba8386d0e037fdad07b55820eae in 497ms, sequenceid=278, compaction requested=false 2024-11-21T13:25:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:39,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:39,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-21T13:25:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-21T13:25:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,732 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-21T13:25:39,732 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8840 sec 2024-11-21T13:25:39,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,734 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.8920 sec 2024-11-21T13:25:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-21T13:25:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,000 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,001 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:40,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-21T13:25:40,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,003 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T13:25:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,004 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:40,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:40,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T13:25:40,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,140 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:40,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:25:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T13:25:40,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:40,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121038e4b0c811849e1815f3618ffe799a9_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195540138/Put/seqid=0 2024-11-21T13:25:40,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195600217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195600228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195600231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195600233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,250 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195600230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742277_1453 (size=25158) 2024-11-21T13:25:40,281 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:40,295 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121038e4b0c811849e1815f3618ffe799a9_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121038e4b0c811849e1815f3618ffe799a9_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:40,296 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/a008d8bf839b437d96e320239ad31ef9, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:40,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/a008d8bf839b437d96e320239ad31ef9 is 175, key is test_row_0/A:col10/1732195540138/Put/seqid=0 2024-11-21T13:25:40,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742276_1452 (size=74795) 2024-11-21T13:25:40,299 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/a008d8bf839b437d96e320239ad31ef9 2024-11-21T13:25:40,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T13:25:40,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:40,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/e0fdb45567134c71bff9ebae7fff0b30 is 50, key is test_row_0/B:col10/1732195540138/Put/seqid=0 2024-11-21T13:25:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T13:25:40,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195600339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,349 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195600349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195600361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195600361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195600361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742278_1454 (size=12301) 2024-11-21T13:25:40,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T13:25:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195600559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,560 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195600579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195600580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195600580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,585 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195600580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T13:25:40,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:40,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T13:25:40,771 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/e0fdb45567134c71bff9ebae7fff0b30 2024-11-21T13:25:40,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T13:25:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/0799fa74ef1a40699876271f597a7fd2 is 50, key is test_row_0/C:col10/1732195540138/Put/seqid=0 2024-11-21T13:25:40,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742279_1455 (size=12301) 2024-11-21T13:25:40,833 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/0799fa74ef1a40699876271f597a7fd2 2024-11-21T13:25:40,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/a008d8bf839b437d96e320239ad31ef9 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a008d8bf839b437d96e320239ad31ef9 2024-11-21T13:25:40,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a008d8bf839b437d96e320239ad31ef9, entries=400, sequenceid=290, filesize=73.0 K 2024-11-21T13:25:40,847 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/e0fdb45567134c71bff9ebae7fff0b30 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e0fdb45567134c71bff9ebae7fff0b30 2024-11-21T13:25:40,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e0fdb45567134c71bff9ebae7fff0b30, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T13:25:40,858 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/0799fa74ef1a40699876271f597a7fd2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/0799fa74ef1a40699876271f597a7fd2 2024-11-21T13:25:40,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/0799fa74ef1a40699876271f597a7fd2, entries=150, sequenceid=290, filesize=12.0 K 2024-11-21T13:25:40,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 6e6e9ba8386d0e037fdad07b55820eae in 723ms, sequenceid=290, compaction requested=true 2024-11-21T13:25:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:40,863 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:40,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-21T13:25:40,863 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:40,864 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 137769 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:40,864 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/A is initiating minor compaction (all files) 2024-11-21T13:25:40,865 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/A in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,865 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/4193922f88f944dc873d08df7f9cfbcf, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8831da03cc874e018ab6e5e750d8a535, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a008d8bf839b437d96e320239ad31ef9] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=134.5 K 2024-11-21T13:25:40,865 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,865 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/4193922f88f944dc873d08df7f9cfbcf, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8831da03cc874e018ab6e5e750d8a535, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a008d8bf839b437d96e320239ad31ef9] 2024-11-21T13:25:40,865 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:40,865 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/B is initiating minor compaction (all files) 2024-11-21T13:25:40,865 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/B in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,865 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/ec384c27587a48c7a1c99effb44a0689, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/65d2b88a80194d81861363559a40bc25, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e0fdb45567134c71bff9ebae7fff0b30] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=36.5 K 2024-11-21T13:25:40,866 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4193922f88f944dc873d08df7f9cfbcf, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732195537439 2024-11-21T13:25:40,866 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8831da03cc874e018ab6e5e750d8a535, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732195537822 2024-11-21T13:25:40,866 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting ec384c27587a48c7a1c99effb44a0689, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732195537439 2024-11-21T13:25:40,867 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 65d2b88a80194d81861363559a40bc25, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732195537822 2024-11-21T13:25:40,867 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting a008d8bf839b437d96e320239ad31ef9, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732195540105 2024-11-21T13:25:40,868 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e0fdb45567134c71bff9ebae7fff0b30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732195540112 2024-11-21T13:25:40,880 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:40,880 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T13:25:40,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:40,903 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#B#compaction#377 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:40,903 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/49f60c23f4f94836a09a2f670c38a750 is 50, key is test_row_0/B:col10/1732195540138/Put/seqid=0 2024-11-21T13:25:40,914 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:40,913 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195600898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195600899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195600901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195600902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:40,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195600910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:40,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:40,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:40,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T13:25:40,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:40,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:40,947 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112194970e05795a4c699743d05a8db76405_6e6e9ba8386d0e037fdad07b55820eae store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:40,951 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112194970e05795a4c699743d05a8db76405_6e6e9ba8386d0e037fdad07b55820eae, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:40,951 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112194970e05795a4c699743d05a8db76405_6e6e9ba8386d0e037fdad07b55820eae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:40,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121239023d8b35b4e6db3ea96ccf219884d_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195540227/Put/seqid=0 2024-11-21T13:25:40,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742280_1456 (size=13017) 2024-11-21T13:25:40,983 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/49f60c23f4f94836a09a2f670c38a750 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/49f60c23f4f94836a09a2f670c38a750 2024-11-21T13:25:40,987 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/B of 6e6e9ba8386d0e037fdad07b55820eae into 49f60c23f4f94836a09a2f670c38a750(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:40,987 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:40,988 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/B, priority=13, startTime=1732195540863; duration=0sec 2024-11-21T13:25:40,988 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:40,988 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:B 2024-11-21T13:25:40,988 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:40,989 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:40,989 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/C is initiating minor compaction (all files) 2024-11-21T13:25:40,989 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/C in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:40,989 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/a9731f49a5394c47b85d9f8de12a1578, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/8cc40f607b3647f4b7f2d961e549351e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/0799fa74ef1a40699876271f597a7fd2] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=36.5 K 2024-11-21T13:25:40,989 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting a9731f49a5394c47b85d9f8de12a1578, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1732195537439 2024-11-21T13:25:40,989 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cc40f607b3647f4b7f2d961e549351e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1732195537822 2024-11-21T13:25:40,990 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 0799fa74ef1a40699876271f597a7fd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732195540112 2024-11-21T13:25:40,998 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:40,999 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/8639c51dd789436794aacc157f5121c7 is 50, key is test_row_0/C:col10/1732195540138/Put/seqid=0 2024-11-21T13:25:41,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742281_1457 (size=4469) 2024-11-21T13:25:41,013 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#A#compaction#378 average throughput is 0.25 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:41,016 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/3262a501ef9a4b089dc34a8d9387f8fb is 175, key is test_row_0/A:col10/1732195540138/Put/seqid=0 2024-11-21T13:25:41,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195601020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,030 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195601020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195601020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195601020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742282_1458 (size=14994) 2024-11-21T13:25:41,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:41,039 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195601029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,042 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121239023d8b35b4e6db3ea96ccf219884d_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121239023d8b35b4e6db3ea96ccf219884d_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:41,043 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/0cbdba9ecca24824942ff0cb2886ef44, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:41,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/0cbdba9ecca24824942ff0cb2886ef44 is 175, key is test_row_0/A:col10/1732195540227/Put/seqid=0 2024-11-21T13:25:41,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742283_1459 (size=13017) 2024-11-21T13:25:41,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T13:25:41,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:41,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:41,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:41,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:41,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:41,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:41,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742284_1460 (size=31971) 2024-11-21T13:25:41,126 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/3262a501ef9a4b089dc34a8d9387f8fb as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/3262a501ef9a4b089dc34a8d9387f8fb 2024-11-21T13:25:41,136 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/A of 6e6e9ba8386d0e037fdad07b55820eae into 3262a501ef9a4b089dc34a8d9387f8fb(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:41,136 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:41,136 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/A, priority=13, startTime=1732195540863; duration=0sec 2024-11-21T13:25:41,136 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:41,136 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:A 2024-11-21T13:25:41,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742285_1461 (size=39949) 2024-11-21T13:25:41,137 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/0cbdba9ecca24824942ff0cb2886ef44 2024-11-21T13:25:41,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T13:25:41,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/24e79e4d912140cda3bec61c59d8bb3a is 50, key is test_row_0/B:col10/1732195540227/Put/seqid=0 2024-11-21T13:25:41,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742286_1462 (size=12301) 2024-11-21T13:25:41,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/24e79e4d912140cda3bec61c59d8bb3a 2024-11-21T13:25:41,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/fc2398beeaab406a8a31c5f2a177e01c is 50, key is test_row_0/C:col10/1732195540227/Put/seqid=0 2024-11-21T13:25:41,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742287_1463 (size=12301) 2024-11-21T13:25:41,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T13:25:41,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:41,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:41,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:41,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:41,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:41,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195601240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,244 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:41,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195601240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195601241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195601241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195601251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,256 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T13:25:41,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:41,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:41,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:41,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:41,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:41,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:41,486 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/8639c51dd789436794aacc157f5121c7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/8639c51dd789436794aacc157f5121c7 2024-11-21T13:25:41,490 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/C of 6e6e9ba8386d0e037fdad07b55820eae into 8639c51dd789436794aacc157f5121c7(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:41,490 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:41,490 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/C, priority=13, startTime=1732195540863; duration=0sec 2024-11-21T13:25:41,490 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:41,491 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:C 2024-11-21T13:25:41,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T13:25:41,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:41,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:41,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:41,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:41,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:41,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195601550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,558 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195601559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195601559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,565 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195601559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,566 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195601570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:41,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:41,614 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/fc2398beeaab406a8a31c5f2a177e01c 2024-11-21T13:25:41,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/0cbdba9ecca24824942ff0cb2886ef44 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0cbdba9ecca24824942ff0cb2886ef44 2024-11-21T13:25:41,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0cbdba9ecca24824942ff0cb2886ef44, entries=200, sequenceid=316, filesize=39.0 K 2024-11-21T13:25:41,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/24e79e4d912140cda3bec61c59d8bb3a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/24e79e4d912140cda3bec61c59d8bb3a 2024-11-21T13:25:41,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/24e79e4d912140cda3bec61c59d8bb3a, entries=150, sequenceid=316, filesize=12.0 K 2024-11-21T13:25:41,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/fc2398beeaab406a8a31c5f2a177e01c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/fc2398beeaab406a8a31c5f2a177e01c 2024-11-21T13:25:41,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/fc2398beeaab406a8a31c5f2a177e01c, entries=150, sequenceid=316, filesize=12.0 K 2024-11-21T13:25:41,648 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 6e6e9ba8386d0e037fdad07b55820eae in 768ms, sequenceid=316, compaction requested=false 2024-11-21T13:25:41,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:41,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-21T13:25:41,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:41,708 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T13:25:41,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b7d42cb33ef74c61a9bd85c2d5be7b39_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195540908/Put/seqid=0 2024-11-21T13:25:41,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742288_1464 (size=12454) 2024-11-21T13:25:42,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:42,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:42,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195602128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T13:25:42,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195602138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195602137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195602139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195602140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:42,194 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b7d42cb33ef74c61a9bd85c2d5be7b39_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b7d42cb33ef74c61a9bd85c2d5be7b39_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:42,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/69c7559837644d659b6424e43c319148, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:42,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/69c7559837644d659b6424e43c319148 is 175, key is test_row_0/A:col10/1732195540908/Put/seqid=0 2024-11-21T13:25:42,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742289_1465 (size=31255) 2024-11-21T13:25:42,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195602249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,257 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195602259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195602259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195602259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,266 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,267 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195602260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195602469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195602470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195602470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,476 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,477 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195602472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195602472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,652 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/69c7559837644d659b6424e43c319148 2024-11-21T13:25:42,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/263b594ce2d64674b0b0f8dd4b7ba849 is 50, key is test_row_0/B:col10/1732195540908/Put/seqid=0 2024-11-21T13:25:42,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742290_1466 (size=12301) 2024-11-21T13:25:42,720 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/263b594ce2d64674b0b0f8dd4b7ba849 2024-11-21T13:25:42,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/4b6971d8b1ee417b99274509ff5d3e9d is 50, key is test_row_0/C:col10/1732195540908/Put/seqid=0 2024-11-21T13:25:42,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742291_1467 (size=12301) 2024-11-21T13:25:42,757 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/4b6971d8b1ee417b99274509ff5d3e9d 2024-11-21T13:25:42,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/69c7559837644d659b6424e43c319148 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/69c7559837644d659b6424e43c319148 2024-11-21T13:25:42,786 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/69c7559837644d659b6424e43c319148, entries=150, sequenceid=329, filesize=30.5 K 2024-11-21T13:25:42,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/263b594ce2d64674b0b0f8dd4b7ba849 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/263b594ce2d64674b0b0f8dd4b7ba849 2024-11-21T13:25:42,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195602779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195602780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,794 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/263b594ce2d64674b0b0f8dd4b7ba849, entries=150, sequenceid=329, filesize=12.0 K 2024-11-21T13:25:42,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/4b6971d8b1ee417b99274509ff5d3e9d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/4b6971d8b1ee417b99274509ff5d3e9d 2024-11-21T13:25:42,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195602790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:42,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195602791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195602791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:42,806 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:42,807 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/4b6971d8b1ee417b99274509ff5d3e9d, entries=150, sequenceid=329, filesize=12.0 K 2024-11-21T13:25:42,808 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 6e6e9ba8386d0e037fdad07b55820eae in 1099ms, sequenceid=329, compaction requested=true 2024-11-21T13:25:42,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:42,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:42,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-21T13:25:42,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-21T13:25:42,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-21T13:25:42,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8050 sec 2024-11-21T13:25:42,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.8110 sec 2024-11-21T13:25:43,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:43,306 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:43,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-21T13:25:43,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195603318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195603319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219f6d25ee18644ec7aa2428fb13ad71c4_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195542126/Put/seqid=0 2024-11-21T13:25:43,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195603319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195603320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195603322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,324 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742292_1468 (size=14994) 2024-11-21T13:25:43,344 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:43,351 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219f6d25ee18644ec7aa2428fb13ad71c4_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219f6d25ee18644ec7aa2428fb13ad71c4_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:43,352 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/a9e1c18c2c56458c9a45e9b1985bdf87, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:43,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/a9e1c18c2c56458c9a45e9b1985bdf87 is 175, key is test_row_0/A:col10/1732195542126/Put/seqid=0 2024-11-21T13:25:43,357 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742293_1469 (size=39949) 2024-11-21T13:25:43,358 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=357, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/a9e1c18c2c56458c9a45e9b1985bdf87 2024-11-21T13:25:43,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/c5f3f48167af493d97bb555ab6d26662 is 50, key is test_row_0/B:col10/1732195542126/Put/seqid=0 2024-11-21T13:25:43,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742294_1470 (size=12301) 2024-11-21T13:25:43,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195603429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195603430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195603430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,437 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195603430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,438 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195603640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195603640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195603649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,661 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195603649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/c5f3f48167af493d97bb555ab6d26662 2024-11-21T13:25:43,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/b97e9d38a7d349b0a2426b246251e4f5 is 50, key is test_row_0/C:col10/1732195542126/Put/seqid=0 2024-11-21T13:25:43,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742295_1471 (size=12301) 2024-11-21T13:25:43,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195603959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195603959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195603970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195603970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:43,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-21T13:25:44,170 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:44,170 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:44,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-21T13:25:44,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T13:25:44,172 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:44,173 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:44,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:44,254 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=357 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/b97e9d38a7d349b0a2426b246251e4f5 2024-11-21T13:25:44,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/a9e1c18c2c56458c9a45e9b1985bdf87 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a9e1c18c2c56458c9a45e9b1985bdf87 2024-11-21T13:25:44,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a9e1c18c2c56458c9a45e9b1985bdf87, entries=200, sequenceid=357, filesize=39.0 K 2024-11-21T13:25:44,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/c5f3f48167af493d97bb555ab6d26662 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/c5f3f48167af493d97bb555ab6d26662 2024-11-21T13:25:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/c5f3f48167af493d97bb555ab6d26662, entries=150, sequenceid=357, filesize=12.0 K 2024-11-21T13:25:44,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/b97e9d38a7d349b0a2426b246251e4f5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/b97e9d38a7d349b0a2426b246251e4f5 2024-11-21T13:25:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/b97e9d38a7d349b0a2426b246251e4f5, entries=150, sequenceid=357, filesize=12.0 K 2024-11-21T13:25:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,279 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 6e6e9ba8386d0e037fdad07b55820eae in 973ms, sequenceid=357, compaction requested=true 2024-11-21T13:25:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:44,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:44,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:44,280 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:25:44,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:44,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:44,280 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:25:44,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:44,280 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T13:25:44,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,285 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:25:44,286 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/B is initiating minor compaction (all files) 2024-11-21T13:25:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,286 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:25:44,286 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/B in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:44,286 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/A is initiating minor compaction (all files) 2024-11-21T13:25:44,286 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/A in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:44,286 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/49f60c23f4f94836a09a2f670c38a750, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/24e79e4d912140cda3bec61c59d8bb3a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/263b594ce2d64674b0b0f8dd4b7ba849, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/c5f3f48167af493d97bb555ab6d26662] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=48.8 K 2024-11-21T13:25:44,286 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/3262a501ef9a4b089dc34a8d9387f8fb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0cbdba9ecca24824942ff0cb2886ef44, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/69c7559837644d659b6424e43c319148, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a9e1c18c2c56458c9a45e9b1985bdf87] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=139.8 K 2024-11-21T13:25:44,286 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:44,286 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/3262a501ef9a4b089dc34a8d9387f8fb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0cbdba9ecca24824942ff0cb2886ef44, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/69c7559837644d659b6424e43c319148, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a9e1c18c2c56458c9a45e9b1985bdf87] 2024-11-21T13:25:44,286 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 49f60c23f4f94836a09a2f670c38a750, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732195540112 2024-11-21T13:25:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,286 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3262a501ef9a4b089dc34a8d9387f8fb, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732195540112 2024-11-21T13:25:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,286 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 24e79e4d912140cda3bec61c59d8bb3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732195540212 2024-11-21T13:25:44,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,287 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cbdba9ecca24824942ff0cb2886ef44, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732195540212 2024-11-21T13:25:44,287 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 263b594ce2d64674b0b0f8dd4b7ba849, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732195540898 2024-11-21T13:25:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,287 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69c7559837644d659b6424e43c319148, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732195540898 2024-11-21T13:25:44,288 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting a9e1c18c2c56458c9a45e9b1985bdf87, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732195542126 2024-11-21T13:25:44,288 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting c5f3f48167af493d97bb555ab6d26662, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732195542126 2024-11-21T13:25:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,298 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,312 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#B#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:44,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,312 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/96ac9b9e9c83405d8a2af7035e6abe3f is 50, key is test_row_0/B:col10/1732195542126/Put/seqid=0 2024-11-21T13:25:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,314 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121ba3e9e9cfc814f8388ebe7b19c6455b9_6e6e9ba8386d0e037fdad07b55820eae store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,316 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121ba3e9e9cfc814f8388ebe7b19c6455b9_6e6e9ba8386d0e037fdad07b55820eae, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:44,316 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ba3e9e9cfc814f8388ebe7b19c6455b9_6e6e9ba8386d0e037fdad07b55820eae because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-21T13:25:44,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:44,328 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T13:25:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742297_1473 (size=4469) 2024-11-21T13:25:44,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742296_1472 (size=13153) 2024-11-21T13:25:44,335 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#A#compaction#389 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:44,336 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/e8c6c4db964543c99035b746d23b600e is 175, key is test_row_0/A:col10/1732195542126/Put/seqid=0 2024-11-21T13:25:44,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211ac9369262e1478ca0197aba5a52fb70_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195543318/Put/seqid=0 2024-11-21T13:25:44,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,342 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/96ac9b9e9c83405d8a2af7035e6abe3f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/96ac9b9e9c83405d8a2af7035e6abe3f 2024-11-21T13:25:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,352 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/B of 6e6e9ba8386d0e037fdad07b55820eae into 96ac9b9e9c83405d8a2af7035e6abe3f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:44,352 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,352 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/B, priority=12, startTime=1732195544280; duration=0sec 2024-11-21T13:25:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,352 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:44,352 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:B 2024-11-21T13:25:44,352 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:25:44,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,354 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:25:44,354 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 6e6e9ba8386d0e037fdad07b55820eae/C is initiating minor compaction (all files) 2024-11-21T13:25:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,354 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6e6e9ba8386d0e037fdad07b55820eae/C in TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:44,354 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/8639c51dd789436794aacc157f5121c7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/fc2398beeaab406a8a31c5f2a177e01c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/4b6971d8b1ee417b99274509ff5d3e9d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/b97e9d38a7d349b0a2426b246251e4f5] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp, totalSize=48.8 K 2024-11-21T13:25:44,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,354 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 8639c51dd789436794aacc157f5121c7, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732195540112 2024-11-21T13:25:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,355 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting fc2398beeaab406a8a31c5f2a177e01c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1732195540212 2024-11-21T13:25:44,355 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b6971d8b1ee417b99274509ff5d3e9d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732195540898 2024-11-21T13:25:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742298_1474 (size=32107) 2024-11-21T13:25:44,356 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b97e9d38a7d349b0a2426b246251e4f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=357, earliestPutTs=1732195542126 2024-11-21T13:25:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,360 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/e8c6c4db964543c99035b746d23b600e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/e8c6c4db964543c99035b746d23b600e 2024-11-21T13:25:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,365 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/A of 6e6e9ba8386d0e037fdad07b55820eae into e8c6c4db964543c99035b746d23b600e(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:44,365 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:44,365 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/A, priority=12, startTime=1732195544279; duration=0sec 2024-11-21T13:25:44,366 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:44,366 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:A 2024-11-21T13:25:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,367 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6e6e9ba8386d0e037fdad07b55820eae#C#compaction#392 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:44,367 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/62f3aec7a87944649679391c911814b2 is 50, key is test_row_0/C:col10/1732195542126/Put/seqid=0 2024-11-21T13:25:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742299_1475 (size=9914) 2024-11-21T13:25:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742300_1476 (size=13153) 2024-11-21T13:25:44,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T13:25:44,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195604513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195604513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195604514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195604514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195604528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195604629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195604629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195604629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195604629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195604639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:44,791 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211ac9369262e1478ca0197aba5a52fb70_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211ac9369262e1478ca0197aba5a52fb70_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:44,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/5e744f3b2a644ea6870ab068aaef0be5, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:44,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/5e744f3b2a644ea6870ab068aaef0be5 is 175, key is test_row_0/A:col10/1732195543318/Put/seqid=0 2024-11-21T13:25:44,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742301_1477 (size=22561) 2024-11-21T13:25:44,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T13:25:44,800 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=365, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/5e744f3b2a644ea6870ab068aaef0be5 2024-11-21T13:25:44,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/e174437e21474ea1abf8a0d473af7470 is 50, key is test_row_0/B:col10/1732195543318/Put/seqid=0 2024-11-21T13:25:44,810 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/62f3aec7a87944649679391c911814b2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/62f3aec7a87944649679391c911814b2 2024-11-21T13:25:44,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742302_1478 (size=9857) 2024-11-21T13:25:44,814 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/e174437e21474ea1abf8a0d473af7470 2024-11-21T13:25:44,817 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6e6e9ba8386d0e037fdad07b55820eae/C of 6e6e9ba8386d0e037fdad07b55820eae into 62f3aec7a87944649679391c911814b2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:44,817 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:44,817 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., storeName=6e6e9ba8386d0e037fdad07b55820eae/C, priority=12, startTime=1732195544280; duration=0sec 2024-11-21T13:25:44,817 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:44,817 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:C 2024-11-21T13:25:44,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/9b9fc883914b4fc1890d164b70abbb3f is 50, key is test_row_0/C:col10/1732195543318/Put/seqid=0 2024-11-21T13:25:44,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742303_1479 (size=9857) 2024-11-21T13:25:44,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195604839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195604839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,844 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195604849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195604849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:44,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195604859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:44,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:44,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:45,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:45,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195605149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:45,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195605149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:45,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:45,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195605159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:45,166 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195605159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,166 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:45,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:45,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195605169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,176 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:45,239 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/9b9fc883914b4fc1890d164b70abbb3f 2024-11-21T13:25:45,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/5e744f3b2a644ea6870ab068aaef0be5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5e744f3b2a644ea6870ab068aaef0be5 2024-11-21T13:25:45,260 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5e744f3b2a644ea6870ab068aaef0be5, entries=100, sequenceid=365, filesize=22.0 K 2024-11-21T13:25:45,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/e174437e21474ea1abf8a0d473af7470 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e174437e21474ea1abf8a0d473af7470 2024-11-21T13:25:45,265 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e174437e21474ea1abf8a0d473af7470, entries=100, sequenceid=365, filesize=9.6 K 2024-11-21T13:25:45,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/9b9fc883914b4fc1890d164b70abbb3f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/9b9fc883914b4fc1890d164b70abbb3f 2024-11-21T13:25:45,274 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/9b9fc883914b4fc1890d164b70abbb3f, entries=100, sequenceid=365, filesize=9.6 K 2024-11-21T13:25:45,275 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 6e6e9ba8386d0e037fdad07b55820eae in 947ms, sequenceid=365, compaction requested=false 2024-11-21T13:25:45,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:45,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:45,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-21T13:25:45,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-21T13:25:45,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-21T13:25:45,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1060 sec 2024-11-21T13:25:45,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.1120 sec 2024-11-21T13:25:45,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-21T13:25:45,309 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:45,310 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:45,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-21T13:25:45,313 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:45,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T13:25:45,315 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:45,315 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:45,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T13:25:45,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-21T13:25:45,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:45,467 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-21T13:25:45,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216c6b1e692f3e403099291ad6d008de78_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195544512/Put/seqid=0 2024-11-21T13:25:45,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742304_1480 (size=12454) 2024-11-21T13:25:45,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,489 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216c6b1e692f3e403099291ad6d008de78_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216c6b1e692f3e403099291ad6d008de78_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:45,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/17f0c248bccd411f9500149f35709729, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:45,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/17f0c248bccd411f9500149f35709729 is 175, key is test_row_0/A:col10/1732195544512/Put/seqid=0 2024-11-21T13:25:45,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742305_1481 (size=31255) 2024-11-21T13:25:45,513 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/17f0c248bccd411f9500149f35709729 2024-11-21T13:25:45,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/e5255965718141a59212201c8ca8eb5f is 50, key is test_row_0/B:col10/1732195544512/Put/seqid=0 2024-11-21T13:25:45,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742306_1482 (size=12301) 2024-11-21T13:25:45,542 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/e5255965718141a59212201c8ca8eb5f 2024-11-21T13:25:45,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/cb6fcb63e3ff415d9350704379caff64 is 50, key is test_row_0/C:col10/1732195544512/Put/seqid=0 2024-11-21T13:25:45,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742307_1483 (size=12301) 2024-11-21T13:25:45,570 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/cb6fcb63e3ff415d9350704379caff64 2024-11-21T13:25:45,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/17f0c248bccd411f9500149f35709729 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/17f0c248bccd411f9500149f35709729 2024-11-21T13:25:45,580 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/17f0c248bccd411f9500149f35709729, entries=150, sequenceid=396, filesize=30.5 K 2024-11-21T13:25:45,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/e5255965718141a59212201c8ca8eb5f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e5255965718141a59212201c8ca8eb5f 2024-11-21T13:25:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,584 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e5255965718141a59212201c8ca8eb5f, entries=150, sequenceid=396, filesize=12.0 K 2024-11-21T13:25:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/cb6fcb63e3ff415d9350704379caff64 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/cb6fcb63e3ff415d9350704379caff64 2024-11-21T13:25:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,589 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/cb6fcb63e3ff415d9350704379caff64, entries=150, sequenceid=396, filesize=12.0 K 2024-11-21T13:25:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,590 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=0 B/0 for 6e6e9ba8386d0e037fdad07b55820eae in 123ms, sequenceid=396, compaction requested=true 2024-11-21T13:25:45,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:45,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:45,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-21T13:25:45,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-21T13:25:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-21T13:25:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,594 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 277 msec 2024-11-21T13:25:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 285 msec 2024-11-21T13:25:45,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-21T13:25:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,630 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,630 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:45,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-21T13:25:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T13:25:45,633 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,633 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:45,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T13:25:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-21T13:25:45,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,786 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-21T13:25:45,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b0d5f9d25b1c4df1bc192d95c743aa35_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_1/A:col10/1732195545779/Put/seqid=0 2024-11-21T13:25:45,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. as already flushing 2024-11-21T13:25:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742308_1484 (size=17534) 2024-11-21T13:25:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,868 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b0d5f9d25b1c4df1bc192d95c743aa35_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b0d5f9d25b1c4df1bc192d95c743aa35_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:45,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/e626a5fdf1f94931aca856c1d7048c98, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:45,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/e626a5fdf1f94931aca856c1d7048c98 is 175, key is test_row_1/A:col10/1732195545779/Put/seqid=0 2024-11-21T13:25:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:45,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742309_1485 (size=48635) 2024-11-21T13:25:45,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:45,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195605921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,935 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:45,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:45,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195605931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:45,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:45,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195605931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T13:25:45,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:45,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195605933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:45,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195605934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195606040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,045 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195606049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195606049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,064 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195606059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195606059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195606249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,255 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T13:25:46,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195606259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195606259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195606275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,284 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195606275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,311 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/e626a5fdf1f94931aca856c1d7048c98 2024-11-21T13:25:46,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/bc40b01e9d0b465692bfb9577cd883be is 50, key is test_row_1/B:col10/1732195545779/Put/seqid=0 2024-11-21T13:25:46,325 INFO [Thread-1691 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1691. 2024-11-21T13:25:46,325 DEBUG [Thread-1691 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:46,325 DEBUG [Thread-1691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:46,325 DEBUG [Thread-1691 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:46,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:46,325 INFO [Thread-1685 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1685. 2024-11-21T13:25:46,325 DEBUG [Thread-1685 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:46,325 DEBUG [Thread-1685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:46,325 DEBUG [Thread-1685 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:46,325 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:46,326 INFO [Thread-1689 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1689. 2024-11-21T13:25:46,326 DEBUG [Thread-1689 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:46,326 DEBUG [Thread-1689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:46,326 DEBUG [Thread-1689 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:46,326 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:46,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742310_1486 (size=9857) 2024-11-21T13:25:46,334 INFO [Thread-1687 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1687. 2024-11-21T13:25:46,334 DEBUG [Thread-1687 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:46,334 INFO [Thread-1683 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1683. 2024-11-21T13:25:46,334 DEBUG [Thread-1687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:46,334 DEBUG [Thread-1683 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:46,334 DEBUG [Thread-1683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:46,334 DEBUG [Thread-1687 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:46,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:46,334 DEBUG [Thread-1683 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:46,334 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:46,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195606560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195606579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195606579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195606589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:46,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195606600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:46,730 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/bc40b01e9d0b465692bfb9577cd883be 2024-11-21T13:25:46,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/6901366b4f384836b6586d1df9bcdf8f is 50, key is test_row_1/C:col10/1732195545779/Put/seqid=0 2024-11-21T13:25:46,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742311_1487 (size=9857) 2024-11-21T13:25:46,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T13:25:47,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:47,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55074 deadline: 1732195607070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:47,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55028 deadline: 1732195607090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:47,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54992 deadline: 1732195607090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:47,092 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:47,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:47,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55068 deadline: 1732195607099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:47,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:47,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:47,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55016 deadline: 1732195607110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:47,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:47,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:25:47,142 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/6901366b4f384836b6586d1df9bcdf8f 2024-11-21T13:25:47,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/e626a5fdf1f94931aca856c1d7048c98 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/e626a5fdf1f94931aca856c1d7048c98 2024-11-21T13:25:47,157 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/e626a5fdf1f94931aca856c1d7048c98, entries=250, sequenceid=404, filesize=47.5 K 2024-11-21T13:25:47,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/bc40b01e9d0b465692bfb9577cd883be as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bc40b01e9d0b465692bfb9577cd883be 2024-11-21T13:25:47,161 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bc40b01e9d0b465692bfb9577cd883be, entries=100, sequenceid=404, filesize=9.6 K 2024-11-21T13:25:47,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/6901366b4f384836b6586d1df9bcdf8f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/6901366b4f384836b6586d1df9bcdf8f 2024-11-21T13:25:47,165 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/6901366b4f384836b6586d1df9bcdf8f, entries=100, sequenceid=404, filesize=9.6 K 2024-11-21T13:25:47,166 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 6e6e9ba8386d0e037fdad07b55820eae in 1380ms, sequenceid=404, compaction requested=true 2024-11-21T13:25:47,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:47,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:47,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-21T13:25:47,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-21T13:25:47,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-21T13:25:47,168 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5340 sec 2024-11-21T13:25:47,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.5380 sec 2024-11-21T13:25:47,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-21T13:25:47,781 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:48,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:48,094 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:48,095 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-21T13:25:48,095 INFO [Thread-1672 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1672. 2024-11-21T13:25:48,095 DEBUG [Thread-1672 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:48,095 DEBUG [Thread-1672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:48,095 DEBUG [Thread-1672 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:48,096 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:48,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121971e6b8ad05c43138bb309284efe952f_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195545932/Put/seqid=0 2024-11-21T13:25:48,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742312_1488 (size=12454) 2024-11-21T13:25:48,111 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:48,112 INFO [Thread-1678 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1678. 2024-11-21T13:25:48,112 DEBUG [Thread-1678 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:48,112 DEBUG [Thread-1678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:48,112 INFO [Thread-1674 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1674. 2024-11-21T13:25:48,112 DEBUG [Thread-1674 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:48,112 DEBUG [Thread-1674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:48,112 DEBUG [Thread-1678 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:48,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:48,112 DEBUG [Thread-1674 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:48,112 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:48,112 INFO [Thread-1676 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1676. 2024-11-21T13:25:48,112 DEBUG [Thread-1676 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:48,112 DEBUG [Thread-1676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:48,113 DEBUG [Thread-1676 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:48,113 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:48,115 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121971e6b8ad05c43138bb309284efe952f_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121971e6b8ad05c43138bb309284efe952f_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:48,116 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/2f5d5d5a265246c2b1f28025b886985a, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:48,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/2f5d5d5a265246c2b1f28025b886985a is 175, key is test_row_0/A:col10/1732195545932/Put/seqid=0 2024-11-21T13:25:48,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742313_1489 (size=31255) 2024-11-21T13:25:48,130 INFO [Thread-1680 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1680. 2024-11-21T13:25:48,130 DEBUG [Thread-1680 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:25:48,130 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:48,130 DEBUG [Thread-1680 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:48,130 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 66 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1214 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3642 rows 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1233 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3699 rows 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1226 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3678 rows 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1240 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3715 rows 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1225 2024-11-21T13:25:48,131 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3675 rows 2024-11-21T13:25:48,131 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T13:25:48,131 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T13:25:48,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:48,131 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:48,131 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:25:48,132 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T13:25:48,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:48,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T13:25:48,135 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195548135"}]},"ts":"1732195548135"} 2024-11-21T13:25:48,136 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T13:25:48,136 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T13:25:48,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T13:25:48,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, UNASSIGN}] 2024-11-21T13:25:48,139 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, UNASSIGN 2024-11-21T13:25:48,140 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=6e6e9ba8386d0e037fdad07b55820eae, regionState=CLOSING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:48,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, UNASSIGN because future has completed 2024-11-21T13:25:48,141 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T13:25:48,141 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:25:48,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T13:25:48,295 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(122): Close 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:48,295 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T13:25:48,296 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1722): Closing 6e6e9ba8386d0e037fdad07b55820eae, disabling compactions & flushes 2024-11-21T13:25:48,296 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:48,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T13:25:48,520 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=433, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/2f5d5d5a265246c2b1f28025b886985a 2024-11-21T13:25:48,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/edc9b0f7a19141bd9d66074a8fed3cf6 is 50, key is test_row_0/B:col10/1732195545932/Put/seqid=0 2024-11-21T13:25:48,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742314_1490 (size=12301) 2024-11-21T13:25:48,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T13:25:48,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/edc9b0f7a19141bd9d66074a8fed3cf6 2024-11-21T13:25:48,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/84669beda3e641e98e95d9d55157e348 is 50, key is test_row_0/C:col10/1732195545932/Put/seqid=0 2024-11-21T13:25:48,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742315_1491 (size=12301) 2024-11-21T13:25:49,230 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T13:25:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T13:25:49,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/84669beda3e641e98e95d9d55157e348 2024-11-21T13:25:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/2f5d5d5a265246c2b1f28025b886985a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/2f5d5d5a265246c2b1f28025b886985a 2024-11-21T13:25:49,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/2f5d5d5a265246c2b1f28025b886985a, entries=150, sequenceid=433, filesize=30.5 K 2024-11-21T13:25:49,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/edc9b0f7a19141bd9d66074a8fed3cf6 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/edc9b0f7a19141bd9d66074a8fed3cf6 2024-11-21T13:25:49,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/edc9b0f7a19141bd9d66074a8fed3cf6, entries=150, sequenceid=433, filesize=12.0 K 2024-11-21T13:25:49,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/84669beda3e641e98e95d9d55157e348 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/84669beda3e641e98e95d9d55157e348 2024-11-21T13:25:49,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/84669beda3e641e98e95d9d55157e348, entries=150, sequenceid=433, filesize=12.0 K 2024-11-21T13:25:49,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 6e6e9ba8386d0e037fdad07b55820eae in 1286ms, sequenceid=433, compaction requested=true 2024-11-21T13:25:49,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6e6e9ba8386d0e037fdad07b55820eae: 2024-11-21T13:25:49,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:49,380 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:49,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:49,380 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. because compaction request was cancelled 2024-11-21T13:25:49,380 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:49,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:49,380 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:A 2024-11-21T13:25:49,380 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:49,380 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. because compaction request was cancelled 2024-11-21T13:25:49,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6e6e9ba8386d0e037fdad07b55820eae:C, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:49,381 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:B 2024-11-21T13:25:49,381 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. after waiting 0 ms 2024-11-21T13:25:49,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:49,381 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. because compaction request was cancelled 2024-11-21T13:25:49,381 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:49,381 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6e6e9ba8386d0e037fdad07b55820eae:C 2024-11-21T13:25:49,381 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(2902): Flushing 6e6e9ba8386d0e037fdad07b55820eae 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-21T13:25:49,387 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216570d9124ca743e285b793394cd20e73_6e6e9ba8386d0e037fdad07b55820eae is 50, key is test_row_0/A:col10/1732195548110/Put/seqid=0 2024-11-21T13:25:49,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742316_1492 (size=12454) 2024-11-21T13:25:49,793 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:49,802 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216570d9124ca743e285b793394cd20e73_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216570d9124ca743e285b793394cd20e73_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:49,803 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/7c8f246e90a14cf0bd3d607e2c321e65, store: [table=TestAcidGuarantees family=A region=6e6e9ba8386d0e037fdad07b55820eae] 2024-11-21T13:25:49,803 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/7c8f246e90a14cf0bd3d607e2c321e65 is 175, key is test_row_0/A:col10/1732195548110/Put/seqid=0 2024-11-21T13:25:49,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742317_1493 (size=31255) 2024-11-21T13:25:50,208 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=440, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/7c8f246e90a14cf0bd3d607e2c321e65 2024-11-21T13:25:50,219 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/bf7623da753b4ab187d72c37792d9b49 is 50, key is test_row_0/B:col10/1732195548110/Put/seqid=0 2024-11-21T13:25:50,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742318_1494 (size=12301) 2024-11-21T13:25:50,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T13:25:50,624 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/bf7623da753b4ab187d72c37792d9b49 2024-11-21T13:25:50,634 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/dfb2e9f34fd346e6b2a5ac629a5d9656 is 50, key is test_row_0/C:col10/1732195548110/Put/seqid=0 2024-11-21T13:25:50,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742319_1495 (size=12301) 2024-11-21T13:25:51,038 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/dfb2e9f34fd346e6b2a5ac629a5d9656 2024-11-21T13:25:51,045 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/A/7c8f246e90a14cf0bd3d607e2c321e65 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/7c8f246e90a14cf0bd3d607e2c321e65 2024-11-21T13:25:51,051 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/7c8f246e90a14cf0bd3d607e2c321e65, entries=150, sequenceid=440, filesize=30.5 K 2024-11-21T13:25:51,052 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/B/bf7623da753b4ab187d72c37792d9b49 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bf7623da753b4ab187d72c37792d9b49 2024-11-21T13:25:51,055 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bf7623da753b4ab187d72c37792d9b49, entries=150, sequenceid=440, filesize=12.0 K 2024-11-21T13:25:51,056 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/.tmp/C/dfb2e9f34fd346e6b2a5ac629a5d9656 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/dfb2e9f34fd346e6b2a5ac629a5d9656 2024-11-21T13:25:51,060 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/dfb2e9f34fd346e6b2a5ac629a5d9656, entries=150, sequenceid=440, filesize=12.0 K 2024-11-21T13:25:51,061 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6e6e9ba8386d0e037fdad07b55820eae in 1680ms, sequenceid=440, compaction requested=true 2024-11-21T13:25:51,061 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8615e316bb964f34ad9fffb17ffb50f4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/f5cf700f1bd44acd856d83c118a9b4cb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5f8089c1bf4b4bd385a2e2c7b4d746af, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/7d793a93f9be4019a1f82480113c8c75, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/47e1041ec44949c49b8ad4e5ed5820dc, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/9c7cc3f86c9d4dd7a943cd8f5496e035, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0e82f7082f4444958301b449a5ec433b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a7876c7264ff4e63b6ca205af6531040, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/e8aeaea3766c4bffbbdf83683b6cc204, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0ecf9b02d93a4b118a7349988d6f02f4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/d47c11254f2644e89b4037f0d2914b34, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/35c2801afeb040b3ad9fee066718a373, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/471cd01c258147688205a5d6a8945517, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/2de5f65aba834691adf1eff5f52c2397, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8541055b1c124b348083426ccf023670, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5e1d757e95a047dea6d06f6bf914af3a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/464ed647de6b493780d3d765be0c0523, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/34703934102e4838b6b357e636263939, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/4193922f88f944dc873d08df7f9cfbcf, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8831da03cc874e018ab6e5e750d8a535, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a008d8bf839b437d96e320239ad31ef9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/3262a501ef9a4b089dc34a8d9387f8fb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0cbdba9ecca24824942ff0cb2886ef44, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/69c7559837644d659b6424e43c319148, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a9e1c18c2c56458c9a45e9b1985bdf87] to archive 2024-11-21T13:25:51,062 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:25:51,064 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8615e316bb964f34ad9fffb17ffb50f4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8615e316bb964f34ad9fffb17ffb50f4 2024-11-21T13:25:51,065 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/f5cf700f1bd44acd856d83c118a9b4cb to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/f5cf700f1bd44acd856d83c118a9b4cb 2024-11-21T13:25:51,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5f8089c1bf4b4bd385a2e2c7b4d746af to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5f8089c1bf4b4bd385a2e2c7b4d746af 2024-11-21T13:25:51,066 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/7d793a93f9be4019a1f82480113c8c75 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/7d793a93f9be4019a1f82480113c8c75 2024-11-21T13:25:51,067 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/47e1041ec44949c49b8ad4e5ed5820dc to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/47e1041ec44949c49b8ad4e5ed5820dc 2024-11-21T13:25:51,068 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/9c7cc3f86c9d4dd7a943cd8f5496e035 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/9c7cc3f86c9d4dd7a943cd8f5496e035 2024-11-21T13:25:51,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0e82f7082f4444958301b449a5ec433b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0e82f7082f4444958301b449a5ec433b 2024-11-21T13:25:51,069 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a7876c7264ff4e63b6ca205af6531040 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a7876c7264ff4e63b6ca205af6531040 2024-11-21T13:25:51,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/e8aeaea3766c4bffbbdf83683b6cc204 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/e8aeaea3766c4bffbbdf83683b6cc204 2024-11-21T13:25:51,070 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0ecf9b02d93a4b118a7349988d6f02f4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0ecf9b02d93a4b118a7349988d6f02f4 2024-11-21T13:25:51,071 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/d47c11254f2644e89b4037f0d2914b34 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/d47c11254f2644e89b4037f0d2914b34 2024-11-21T13:25:51,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/35c2801afeb040b3ad9fee066718a373 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/35c2801afeb040b3ad9fee066718a373 2024-11-21T13:25:51,072 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/471cd01c258147688205a5d6a8945517 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/471cd01c258147688205a5d6a8945517 2024-11-21T13:25:51,073 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/2de5f65aba834691adf1eff5f52c2397 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/2de5f65aba834691adf1eff5f52c2397 2024-11-21T13:25:51,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8541055b1c124b348083426ccf023670 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8541055b1c124b348083426ccf023670 2024-11-21T13:25:51,074 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5e1d757e95a047dea6d06f6bf914af3a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5e1d757e95a047dea6d06f6bf914af3a 2024-11-21T13:25:51,075 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/464ed647de6b493780d3d765be0c0523 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/464ed647de6b493780d3d765be0c0523 2024-11-21T13:25:51,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/34703934102e4838b6b357e636263939 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/34703934102e4838b6b357e636263939 2024-11-21T13:25:51,076 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/4193922f88f944dc873d08df7f9cfbcf to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/4193922f88f944dc873d08df7f9cfbcf 2024-11-21T13:25:51,077 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8831da03cc874e018ab6e5e750d8a535 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/8831da03cc874e018ab6e5e750d8a535 2024-11-21T13:25:51,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a008d8bf839b437d96e320239ad31ef9 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a008d8bf839b437d96e320239ad31ef9 2024-11-21T13:25:51,078 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/3262a501ef9a4b089dc34a8d9387f8fb to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/3262a501ef9a4b089dc34a8d9387f8fb 2024-11-21T13:25:51,079 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0cbdba9ecca24824942ff0cb2886ef44 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/0cbdba9ecca24824942ff0cb2886ef44 2024-11-21T13:25:51,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/69c7559837644d659b6424e43c319148 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/69c7559837644d659b6424e43c319148 2024-11-21T13:25:51,080 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a9e1c18c2c56458c9a45e9b1985bdf87 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/a9e1c18c2c56458c9a45e9b1985bdf87 2024-11-21T13:25:51,081 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bf31e3c5a5c945959e8a0619bced5cbd, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/f0dde327118f4ac089a53f3604bc27d7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/f639681b8f39427dbcef3d6f4781a05c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/b5b4b625e62f4c9da8e3adcdbc81b2b0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/89fae7bb8d2f4e299c41b4b43a7834ed, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/c8de8ab2a1cc4ef9b36d623ba96d1505, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/2564a56e1b784199aa9b3ab6af067c1f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/012fa91ed5fb46c1872a1fe36ac71c6a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bc670dfabd8546fdb6643de98f074a14, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/2d00cd04144a4a7285007ff43a4e03e9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/13901c1ebd004518bd35b3ecc98522bb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/b1ad0abb31a445bcad45689c92a9ef1e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/344cb3bb052c4a7c8cc7caa6e80f34eb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/59aadb2dd3734ed29953561270e67f25, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/d7c7636764d347e48cd002a79ed34ec7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/9c7f52ca2fb54d13bc61e492517f9134, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/8efbd12138584821ba6a76f6179202cc, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/ec384c27587a48c7a1c99effb44a0689, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/ca12e02939ce415ba3065bc5908e4bea, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/65d2b88a80194d81861363559a40bc25, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/49f60c23f4f94836a09a2f670c38a750, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e0fdb45567134c71bff9ebae7fff0b30, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/24e79e4d912140cda3bec61c59d8bb3a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/263b594ce2d64674b0b0f8dd4b7ba849, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/c5f3f48167af493d97bb555ab6d26662] to archive 2024-11-21T13:25:51,082 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:25:51,083 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bf31e3c5a5c945959e8a0619bced5cbd to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bf31e3c5a5c945959e8a0619bced5cbd 2024-11-21T13:25:51,084 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/f0dde327118f4ac089a53f3604bc27d7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/f0dde327118f4ac089a53f3604bc27d7 2024-11-21T13:25:51,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/f639681b8f39427dbcef3d6f4781a05c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/f639681b8f39427dbcef3d6f4781a05c 2024-11-21T13:25:51,085 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/b5b4b625e62f4c9da8e3adcdbc81b2b0 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/b5b4b625e62f4c9da8e3adcdbc81b2b0 2024-11-21T13:25:51,086 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/89fae7bb8d2f4e299c41b4b43a7834ed to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/89fae7bb8d2f4e299c41b4b43a7834ed 2024-11-21T13:25:51,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/c8de8ab2a1cc4ef9b36d623ba96d1505 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/c8de8ab2a1cc4ef9b36d623ba96d1505 2024-11-21T13:25:51,087 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/2564a56e1b784199aa9b3ab6af067c1f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/2564a56e1b784199aa9b3ab6af067c1f 2024-11-21T13:25:51,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/012fa91ed5fb46c1872a1fe36ac71c6a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/012fa91ed5fb46c1872a1fe36ac71c6a 2024-11-21T13:25:51,088 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bc670dfabd8546fdb6643de98f074a14 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bc670dfabd8546fdb6643de98f074a14 2024-11-21T13:25:51,089 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/2d00cd04144a4a7285007ff43a4e03e9 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/2d00cd04144a4a7285007ff43a4e03e9 2024-11-21T13:25:51,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/13901c1ebd004518bd35b3ecc98522bb to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/13901c1ebd004518bd35b3ecc98522bb 2024-11-21T13:25:51,090 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/b1ad0abb31a445bcad45689c92a9ef1e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/b1ad0abb31a445bcad45689c92a9ef1e 2024-11-21T13:25:51,091 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/344cb3bb052c4a7c8cc7caa6e80f34eb to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/344cb3bb052c4a7c8cc7caa6e80f34eb 2024-11-21T13:25:51,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/59aadb2dd3734ed29953561270e67f25 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/59aadb2dd3734ed29953561270e67f25 2024-11-21T13:25:51,092 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/d7c7636764d347e48cd002a79ed34ec7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/d7c7636764d347e48cd002a79ed34ec7 2024-11-21T13:25:51,093 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/9c7f52ca2fb54d13bc61e492517f9134 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/9c7f52ca2fb54d13bc61e492517f9134 2024-11-21T13:25:51,094 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/8efbd12138584821ba6a76f6179202cc to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/8efbd12138584821ba6a76f6179202cc 2024-11-21T13:25:51,095 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/ec384c27587a48c7a1c99effb44a0689 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/ec384c27587a48c7a1c99effb44a0689 2024-11-21T13:25:51,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/ca12e02939ce415ba3065bc5908e4bea to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/ca12e02939ce415ba3065bc5908e4bea 2024-11-21T13:25:51,096 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/65d2b88a80194d81861363559a40bc25 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/65d2b88a80194d81861363559a40bc25 2024-11-21T13:25:51,097 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/49f60c23f4f94836a09a2f670c38a750 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/49f60c23f4f94836a09a2f670c38a750 2024-11-21T13:25:51,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e0fdb45567134c71bff9ebae7fff0b30 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e0fdb45567134c71bff9ebae7fff0b30 2024-11-21T13:25:51,098 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/24e79e4d912140cda3bec61c59d8bb3a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/24e79e4d912140cda3bec61c59d8bb3a 2024-11-21T13:25:51,099 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/263b594ce2d64674b0b0f8dd4b7ba849 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/263b594ce2d64674b0b0f8dd4b7ba849 2024-11-21T13:25:51,100 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/c5f3f48167af493d97bb555ab6d26662 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/c5f3f48167af493d97bb555ab6d26662 2024-11-21T13:25:51,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/b91c6f738fa0426ea82f3794fb8d0f4a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/7c587dc0479740dc921213737a32e16a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/4d53f5db27c64b93ae1fe9036458a988, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/f12f946a320045818cfcfc4745badef2, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/9fa0c7c5f4754b969e325520e8a97925, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/7c95d4f7620b40518dd9496b0955a50a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/234cd13d749341c9a48aab672f0d7b41, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/5d196ab7bcc340d1a4fd80a3db4abd03, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/3555acc83d5c41fdb305d4ce30476a00, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/cd619d938b1244b6b26306086af5db2a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/e6e8e66c9a3a43518751d49847bcabc4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/6c05a7b123044d0dab92b78bbfe4ba58, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/45433c3d724a4adcaf1a5aa66b335f1a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/db9f135a1e3a4e4a8e03ffc8f11eb377, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/e7c6102d3e054038bdfcd29446e2edb4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/97e4a12049cf4f3fb08029785ef1bf99, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/a9b58972c2a84b1d95db3c65a5c8d849, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/a9731f49a5394c47b85d9f8de12a1578, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/fd11084ceb144cb59bbb14248e3e8de8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/8cc40f607b3647f4b7f2d961e549351e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/8639c51dd789436794aacc157f5121c7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/0799fa74ef1a40699876271f597a7fd2, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/fc2398beeaab406a8a31c5f2a177e01c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/4b6971d8b1ee417b99274509ff5d3e9d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/b97e9d38a7d349b0a2426b246251e4f5] to archive 2024-11-21T13:25:51,101 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:25:51,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/b91c6f738fa0426ea82f3794fb8d0f4a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/b91c6f738fa0426ea82f3794fb8d0f4a 2024-11-21T13:25:51,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/7c587dc0479740dc921213737a32e16a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/7c587dc0479740dc921213737a32e16a 2024-11-21T13:25:51,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/4d53f5db27c64b93ae1fe9036458a988 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/4d53f5db27c64b93ae1fe9036458a988 2024-11-21T13:25:51,104 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/f12f946a320045818cfcfc4745badef2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/f12f946a320045818cfcfc4745badef2 2024-11-21T13:25:51,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/9fa0c7c5f4754b969e325520e8a97925 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/9fa0c7c5f4754b969e325520e8a97925 2024-11-21T13:25:51,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/7c95d4f7620b40518dd9496b0955a50a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/7c95d4f7620b40518dd9496b0955a50a 2024-11-21T13:25:51,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/234cd13d749341c9a48aab672f0d7b41 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/234cd13d749341c9a48aab672f0d7b41 2024-11-21T13:25:51,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/5d196ab7bcc340d1a4fd80a3db4abd03 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/5d196ab7bcc340d1a4fd80a3db4abd03 2024-11-21T13:25:51,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/3555acc83d5c41fdb305d4ce30476a00 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/3555acc83d5c41fdb305d4ce30476a00 2024-11-21T13:25:51,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/cd619d938b1244b6b26306086af5db2a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/cd619d938b1244b6b26306086af5db2a 2024-11-21T13:25:51,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/e6e8e66c9a3a43518751d49847bcabc4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/e6e8e66c9a3a43518751d49847bcabc4 2024-11-21T13:25:51,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/6c05a7b123044d0dab92b78bbfe4ba58 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/6c05a7b123044d0dab92b78bbfe4ba58 2024-11-21T13:25:51,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/45433c3d724a4adcaf1a5aa66b335f1a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/45433c3d724a4adcaf1a5aa66b335f1a 2024-11-21T13:25:51,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/db9f135a1e3a4e4a8e03ffc8f11eb377 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/db9f135a1e3a4e4a8e03ffc8f11eb377 2024-11-21T13:25:51,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/e7c6102d3e054038bdfcd29446e2edb4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/e7c6102d3e054038bdfcd29446e2edb4 2024-11-21T13:25:51,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/97e4a12049cf4f3fb08029785ef1bf99 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/97e4a12049cf4f3fb08029785ef1bf99 2024-11-21T13:25:51,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/a9b58972c2a84b1d95db3c65a5c8d849 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/a9b58972c2a84b1d95db3c65a5c8d849 2024-11-21T13:25:51,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/a9731f49a5394c47b85d9f8de12a1578 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/a9731f49a5394c47b85d9f8de12a1578 2024-11-21T13:25:51,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/fd11084ceb144cb59bbb14248e3e8de8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/fd11084ceb144cb59bbb14248e3e8de8 2024-11-21T13:25:51,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/8cc40f607b3647f4b7f2d961e549351e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/8cc40f607b3647f4b7f2d961e549351e 2024-11-21T13:25:51,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/8639c51dd789436794aacc157f5121c7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/8639c51dd789436794aacc157f5121c7 2024-11-21T13:25:51,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/0799fa74ef1a40699876271f597a7fd2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/0799fa74ef1a40699876271f597a7fd2 2024-11-21T13:25:51,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/fc2398beeaab406a8a31c5f2a177e01c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/fc2398beeaab406a8a31c5f2a177e01c 2024-11-21T13:25:51,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/4b6971d8b1ee417b99274509ff5d3e9d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/4b6971d8b1ee417b99274509ff5d3e9d 2024-11-21T13:25:51,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/b97e9d38a7d349b0a2426b246251e4f5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/b97e9d38a7d349b0a2426b246251e4f5 2024-11-21T13:25:51,121 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/recovered.edits/443.seqid, newMaxSeqId=443, maxSeqId=4 2024-11-21T13:25:51,122 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. 2024-11-21T13:25:51,122 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] regionserver.HRegion(1676): Region close journal for 6e6e9ba8386d0e037fdad07b55820eae: Waiting for close lock at 1732195548296Running coprocessor pre-close hooks at 1732195548296Disabling compacts and flushes for region at 1732195548296Disabling writes for close at 1732195549381 (+1085 ms)Obtaining lock to block concurrent updates at 1732195549381Preparing flush snapshotting stores in 6e6e9ba8386d0e037fdad07b55820eae at 1732195549381Finished memstore snapshotting TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732195549381Flushing stores of TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae. at 1732195549381Flushing 6e6e9ba8386d0e037fdad07b55820eae/A: creating writer at 1732195549381Flushing mob file 6e6e9ba8386d0e037fdad07b55820eae/A: appending metadata at 1732195549387 (+6 ms)Flushing mob file 6e6e9ba8386d0e037fdad07b55820eae/A: closing flushed file at 1732195549387Flushing 6e6e9ba8386d0e037fdad07b55820eae/A: appending metadata at 1732195549803 (+416 ms)Flushing 6e6e9ba8386d0e037fdad07b55820eae/A: closing flushed file at 1732195549803Flushing 6e6e9ba8386d0e037fdad07b55820eae/B: creating writer at 1732195550216 (+413 ms)Flushing 6e6e9ba8386d0e037fdad07b55820eae/B: appending metadata at 1732195550219 (+3 ms)Flushing 6e6e9ba8386d0e037fdad07b55820eae/B: closing flushed file at 1732195550219Flushing 6e6e9ba8386d0e037fdad07b55820eae/C: creating writer at 1732195550629 (+410 ms)Flushing 6e6e9ba8386d0e037fdad07b55820eae/C: appending metadata at 1732195550633 (+4 ms)Flushing 6e6e9ba8386d0e037fdad07b55820eae/C: closing flushed file at 1732195550633Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@138d51d2: reopening flushed file at 1732195551043 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e9cd800: reopening flushed file at 1732195551051 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68fe8ee9: reopening flushed file at 1732195551056 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 6e6e9ba8386d0e037fdad07b55820eae in 1680ms, sequenceid=440, compaction requested=true at 1732195551061 (+5 ms)Writing region close event to WAL at 1732195551119 (+58 ms)Running coprocessor post-close hooks at 1732195551122 (+3 ms)Closed at 1732195551122 2024-11-21T13:25:51,123 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=118}] handler.UnassignRegionHandler(157): Closed 6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:51,124 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=6e6e9ba8386d0e037fdad07b55820eae, regionState=CLOSED 2024-11-21T13:25:51,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:25:51,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-21T13:25:51,127 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseRegionProcedure 6e6e9ba8386d0e037fdad07b55820eae, server=4fe6efa1e611,33573,1732195430851 in 2.9850 sec 2024-11-21T13:25:51,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-21T13:25:51,128 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6e6e9ba8386d0e037fdad07b55820eae, UNASSIGN in 2.9890 sec 2024-11-21T13:25:51,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-21T13:25:51,130 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.9920 sec 2024-11-21T13:25:51,131 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195551131"}]},"ts":"1732195551131"} 2024-11-21T13:25:51,132 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T13:25:51,132 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T13:25:51,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 3.0010 sec 2024-11-21T13:25:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-21T13:25:52,301 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:52,302 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T13:25:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:52,307 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=119, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T13:25:52,309 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=119, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:52,312 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,314 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/recovered.edits] 2024-11-21T13:25:52,317 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/17f0c248bccd411f9500149f35709729 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/17f0c248bccd411f9500149f35709729 2024-11-21T13:25:52,319 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/2f5d5d5a265246c2b1f28025b886985a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/2f5d5d5a265246c2b1f28025b886985a 2024-11-21T13:25:52,321 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5e744f3b2a644ea6870ab068aaef0be5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/5e744f3b2a644ea6870ab068aaef0be5 2024-11-21T13:25:52,322 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/7c8f246e90a14cf0bd3d607e2c321e65 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/7c8f246e90a14cf0bd3d607e2c321e65 2024-11-21T13:25:52,324 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/e626a5fdf1f94931aca856c1d7048c98 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/e626a5fdf1f94931aca856c1d7048c98 2024-11-21T13:25:52,325 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/e8c6c4db964543c99035b746d23b600e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/A/e8c6c4db964543c99035b746d23b600e 2024-11-21T13:25:52,328 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/96ac9b9e9c83405d8a2af7035e6abe3f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/96ac9b9e9c83405d8a2af7035e6abe3f 2024-11-21T13:25:52,330 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bc40b01e9d0b465692bfb9577cd883be to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bc40b01e9d0b465692bfb9577cd883be 2024-11-21T13:25:52,331 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bf7623da753b4ab187d72c37792d9b49 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/bf7623da753b4ab187d72c37792d9b49 2024-11-21T13:25:52,332 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e174437e21474ea1abf8a0d473af7470 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e174437e21474ea1abf8a0d473af7470 2024-11-21T13:25:52,333 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e5255965718141a59212201c8ca8eb5f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/e5255965718141a59212201c8ca8eb5f 2024-11-21T13:25:52,335 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/edc9b0f7a19141bd9d66074a8fed3cf6 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/B/edc9b0f7a19141bd9d66074a8fed3cf6 2024-11-21T13:25:52,338 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/62f3aec7a87944649679391c911814b2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/62f3aec7a87944649679391c911814b2 2024-11-21T13:25:52,339 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/6901366b4f384836b6586d1df9bcdf8f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/6901366b4f384836b6586d1df9bcdf8f 2024-11-21T13:25:52,341 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/84669beda3e641e98e95d9d55157e348 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/84669beda3e641e98e95d9d55157e348 2024-11-21T13:25:52,342 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/9b9fc883914b4fc1890d164b70abbb3f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/9b9fc883914b4fc1890d164b70abbb3f 2024-11-21T13:25:52,344 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/cb6fcb63e3ff415d9350704379caff64 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/cb6fcb63e3ff415d9350704379caff64 2024-11-21T13:25:52,346 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/dfb2e9f34fd346e6b2a5ac629a5d9656 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/C/dfb2e9f34fd346e6b2a5ac629a5d9656 2024-11-21T13:25:52,349 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/recovered.edits/443.seqid to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae/recovered.edits/443.seqid 2024-11-21T13:25:52,350 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,350 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T13:25:52,351 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T13:25:52,352 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T13:25:52,356 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121038e4b0c811849e1815f3618ffe799a9_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121038e4b0c811849e1815f3618ffe799a9_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,358 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112108fb758f18014f8ca25dd294267b81c9_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112108fb758f18014f8ca25dd294267b81c9_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,360 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211ac9369262e1478ca0197aba5a52fb70_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411211ac9369262e1478ca0197aba5a52fb70_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,361 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112120acb7f6c5534602823047b07ed2552c_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112120acb7f6c5534602823047b07ed2552c_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,363 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121239023d8b35b4e6db3ea96ccf219884d_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121239023d8b35b4e6db3ea96ccf219884d_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,364 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212c6ec76f5b2a4d97969fd8ef559f217b_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411212c6ec76f5b2a4d97969fd8ef559f217b_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,366 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213539ef12110b4064b028be66bec661a1_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411213539ef12110b4064b028be66bec661a1_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,368 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214961a43a0f324c5bb159cb4561b458ae_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411214961a43a0f324c5bb159cb4561b458ae_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,369 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121532c99728325425b915918b318a3b14c_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121532c99728325425b915918b318a3b14c_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,370 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216570d9124ca743e285b793394cd20e73_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216570d9124ca743e285b793394cd20e73_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,371 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216c6b1e692f3e403099291ad6d008de78_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216c6b1e692f3e403099291ad6d008de78_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,373 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112173d5a609d3494fe8968045e07fd4237e_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112173d5a609d3494fe8968045e07fd4237e_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,374 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217f71d7cf93c9494cb63377ccc3d0416e_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411217f71d7cf93c9494cb63377ccc3d0416e_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,375 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121857cb7d05f314c87bda98e8d4b0f033f_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121857cb7d05f314c87bda98e8d4b0f033f_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,377 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121971e6b8ad05c43138bb309284efe952f_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121971e6b8ad05c43138bb309284efe952f_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,378 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219affe5dcade84fb7aeadd49ffeeb1185_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219affe5dcade84fb7aeadd49ffeeb1185_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,379 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219f6d25ee18644ec7aa2428fb13ad71c4_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219f6d25ee18644ec7aa2428fb13ad71c4_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,380 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a41ab0450fd94a449c058df8ad860205_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a41ab0450fd94a449c058df8ad860205_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,381 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b0d5f9d25b1c4df1bc192d95c743aa35_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b0d5f9d25b1c4df1bc192d95c743aa35_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,382 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b7d42cb33ef74c61a9bd85c2d5be7b39_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121b7d42cb33ef74c61a9bd85c2d5be7b39_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,383 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c15706cc554247a9b774ad4ce8a08c7a_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121c15706cc554247a9b774ad4ce8a08c7a_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,383 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dcda7fcfbf214274a747b6ef4f45ccf1_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121dcda7fcfbf214274a747b6ef4f45ccf1_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,384 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fe074729d08544aebf67cdcb02ee4d92_6e6e9ba8386d0e037fdad07b55820eae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121fe074729d08544aebf67cdcb02ee4d92_6e6e9ba8386d0e037fdad07b55820eae 2024-11-21T13:25:52,384 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T13:25:52,386 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=119, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:52,387 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T13:25:52,389 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T13:25:52,389 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=119, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:52,389 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T13:25:52,389 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732195552389"}]},"ts":"9223372036854775807"} 2024-11-21T13:25:52,391 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T13:25:52,391 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6e6e9ba8386d0e037fdad07b55820eae, NAME => 'TestAcidGuarantees,,1732195523040.6e6e9ba8386d0e037fdad07b55820eae.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T13:25:52,391 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T13:25:52,391 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732195552391"}]},"ts":"9223372036854775807"} 2024-11-21T13:25:52,392 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T13:25:52,393 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=119, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:52,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 90 msec 2024-11-21T13:25:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-21T13:25:52,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T13:25:52,420 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:52,432 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 241), OpenFileDescriptor=467 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=511 (was 508) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2294 (was 2396) 2024-11-21T13:25:52,441 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=467, MaxFileDescriptor=1048576, SystemLoadAverage=511, ProcessCount=11, AvailableMemoryMB=2294 2024-11-21T13:25:52,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T13:25:52,442 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T13:25:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T13:25:52,444 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T13:25:52,444 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:52,444 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 120 2024-11-21T13:25:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-21T13:25:52,445 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T13:25:52,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742320_1496 (size=959) 2024-11-21T13:25:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-21T13:25:52,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-21T13:25:52,857 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48 2024-11-21T13:25:52,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742321_1497 (size=53) 2024-11-21T13:25:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-21T13:25:53,267 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:25:53,267 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 8328be8165f9f405996e915498f7765f, disabling compactions & flushes 2024-11-21T13:25:53,267 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:53,267 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:53,267 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. after waiting 0 ms 2024-11-21T13:25:53,267 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:53,267 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:53,268 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8328be8165f9f405996e915498f7765f: Waiting for close lock at 1732195553267Disabling compacts and flushes for region at 1732195553267Disabling writes for close at 1732195553267Writing region close event to WAL at 1732195553267Closed at 1732195553267 2024-11-21T13:25:53,271 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T13:25:53,271 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732195553271"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732195553271"}]},"ts":"1732195553271"} 2024-11-21T13:25:53,275 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T13:25:53,277 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T13:25:53,278 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195553277"}]},"ts":"1732195553277"} 2024-11-21T13:25:53,280 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T13:25:53,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8328be8165f9f405996e915498f7765f, ASSIGN}] 2024-11-21T13:25:53,281 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8328be8165f9f405996e915498f7765f, ASSIGN 2024-11-21T13:25:53,282 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8328be8165f9f405996e915498f7765f, ASSIGN; state=OFFLINE, location=4fe6efa1e611,33573,1732195430851; forceNewPlan=false, retain=false 2024-11-21T13:25:53,433 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=8328be8165f9f405996e915498f7765f, regionState=OPENING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:53,436 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8328be8165f9f405996e915498f7765f, ASSIGN because future has completed 2024-11-21T13:25:53,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:25:53,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-21T13:25:53,599 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:53,600 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7752): Opening region: {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} 2024-11-21T13:25:53,602 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:53,602 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:25:53,602 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7794): checking encryption for 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:53,602 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(7797): checking classloading for 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:53,604 INFO [StoreOpener-8328be8165f9f405996e915498f7765f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:53,605 INFO [StoreOpener-8328be8165f9f405996e915498f7765f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8328be8165f9f405996e915498f7765f columnFamilyName A 2024-11-21T13:25:53,605 DEBUG [StoreOpener-8328be8165f9f405996e915498f7765f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:53,606 INFO [StoreOpener-8328be8165f9f405996e915498f7765f-1 {}] regionserver.HStore(327): Store=8328be8165f9f405996e915498f7765f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:25:53,606 INFO [StoreOpener-8328be8165f9f405996e915498f7765f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:53,607 INFO [StoreOpener-8328be8165f9f405996e915498f7765f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8328be8165f9f405996e915498f7765f columnFamilyName B 2024-11-21T13:25:53,607 DEBUG [StoreOpener-8328be8165f9f405996e915498f7765f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:53,607 INFO [StoreOpener-8328be8165f9f405996e915498f7765f-1 {}] regionserver.HStore(327): Store=8328be8165f9f405996e915498f7765f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:25:53,607 INFO [StoreOpener-8328be8165f9f405996e915498f7765f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:53,608 INFO [StoreOpener-8328be8165f9f405996e915498f7765f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8328be8165f9f405996e915498f7765f columnFamilyName C 2024-11-21T13:25:53,608 DEBUG [StoreOpener-8328be8165f9f405996e915498f7765f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:25:53,608 INFO [StoreOpener-8328be8165f9f405996e915498f7765f-1 {}] regionserver.HStore(327): Store=8328be8165f9f405996e915498f7765f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:25:53,608 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1038): replaying wal for 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:53,609 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f 2024-11-21T13:25:53,610 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f 2024-11-21T13:25:53,611 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1048): stopping wal replay for 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:53,611 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1060): Cleaning up temporary data for 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:53,611 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T13:25:53,612 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1093): writing seq id for 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:53,614 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T13:25:53,614 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1114): Opened 8328be8165f9f405996e915498f7765f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63825633, jitterRate=-0.048923954367637634}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T13:25:53,614 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:53,615 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegion(1006): Region open journal for 8328be8165f9f405996e915498f7765f: Running coprocessor pre-open hook at 1732195553603Writing region info on filesystem at 1732195553603Initializing all the Stores at 1732195553604 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195553604Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195553604Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195553604Cleaning up temporary data from old regions at 1732195553611 (+7 ms)Running coprocessor post-open hooks at 1732195553614 (+3 ms)Region opened successfully at 1732195553615 (+1 ms) 2024-11-21T13:25:53,616 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., pid=122, masterSystemTime=1732195553592 2024-11-21T13:25:53,617 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:53,617 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=122}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:53,617 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=8328be8165f9f405996e915498f7765f, regionState=OPEN, openSeqNum=2, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:53,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:25:53,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-21T13:25:53,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; OpenRegionProcedure 8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 in 182 msec 2024-11-21T13:25:53,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-21T13:25:53,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8328be8165f9f405996e915498f7765f, ASSIGN in 340 msec 2024-11-21T13:25:53,622 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T13:25:53,622 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195553622"}]},"ts":"1732195553622"} 2024-11-21T13:25:53,623 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T13:25:53,624 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=120, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T13:25:53,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1820 sec 2024-11-21T13:25:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=120 2024-11-21T13:25:54,590 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:54,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77001fc0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:54,594 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:54,596 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c8482d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:54,597 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:54,598 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52620, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:54,599 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a604c06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,599 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:54,601 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:54,601 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48964, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:54,602 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T13:25:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 4fe6efa1e611,46031,1732195430123 2024-11-21T13:25:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1deb14b1 2024-11-21T13:25:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T13:25:54,605 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T13:25:54,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@414548ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:54,606 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:54,607 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:54,607 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7493b157, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:54,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:54,609 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52644, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:54,610 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a5cd4bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,611 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e6f359b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,611 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:54,611 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:54,612 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64157533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:54,612 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:54,613 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:54,614 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:54,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b155af4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,615 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@237707e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,615 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:54,616 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:54,617 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e1f438, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:54,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:54,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:54,618 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:54,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30608ff1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,620 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2375345f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,620 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:54,620 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:54,621 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:54,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34873c98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:54,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:54,623 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:54,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@551178ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@154e075c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,625 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:54,625 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:54,626 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b934704, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:54,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:54,628 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:54,628 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38d9d90a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,629 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a899d0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,629 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:54,630 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:54,631 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45197ef6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:54,631 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:54,633 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:54,634 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26ce4e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a736d8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:54,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:54,636 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30fbc10c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,636 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:54,637 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:54,638 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:54,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6daf441a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@609bc762, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:54,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:54,641 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:54,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:54,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:54,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67c47a3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:54,641 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:54,642 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:54,643 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52750, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:54,643 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4685046d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71dab0db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,644 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:54,644 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:54,645 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c6fb2d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:54,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:54,646 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:54,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52392664, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9eeb91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,647 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:25:54,647 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:25:54,647 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:25:54,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:25:54,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:25:54,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d3f81a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:25:54,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:25:54,648 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:25:54,649 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:25:54,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284f267a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:25:54,650 DEBUG [Thread-2210 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:54,650 DEBUG [Thread-2221 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:54,650 DEBUG [Thread-2219 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:54,650 DEBUG [Thread-2217 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:54,650 DEBUG [Thread-2212 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:54,650 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:54,650 DEBUG [Thread-2204 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:54,651 DEBUG [Thread-2206 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:54,651 DEBUG [Thread-2223 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:54,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-21T13:25:54,652 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:54,652 DEBUG [Thread-2208 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:54,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:54,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:54,652 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:54,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48972, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:54,653 DEBUG [Thread-2215 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:25:54,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:54,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:54,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49000, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T13:25:54,653 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:54,654 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:54,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:54,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:25:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:54,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:25:54,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:54,655 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:54,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49022, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:54,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:54,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:54,656 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49060, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:25:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:54,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:54,658 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:54,659 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:54,660 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2] 2024-11-21T13:25:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:54,663 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:54,663 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:25:54,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/65c22ea15c774a6b88d3f3f516decc98 is 50, key is test_row_0/A:col10/1732195554662/Put/seqid=0 2024-11-21T13:25:54,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742322_1498 (size=12001) 2024-11-21T13:25:54,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195614694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:54,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:54,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195614694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:54,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,697 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:54,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195614695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:54,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195614696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,702 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195614696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,703 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T13:25:54,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195614799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:54,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195614799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:54,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:54,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T13:25:54,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:54,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:25:54,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:54,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:54,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:54,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195614808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195614809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195614809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:54,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:54,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T13:25:54,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:54,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:25:54,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:54,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:54,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:54,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:54,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T13:25:55,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195615009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195615009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195615019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195615019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195615019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/65c22ea15c774a6b88d3f3f516decc98 2024-11-21T13:25:55,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T13:25:55,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:55,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:25:55,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:55,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:55,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:55,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/8567e380ba4e41a189ce389b47cfca36 is 50, key is test_row_0/B:col10/1732195554662/Put/seqid=0 2024-11-21T13:25:55,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742323_1499 (size=12001) 2024-11-21T13:25:55,132 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/8567e380ba4e41a189ce389b47cfca36 2024-11-21T13:25:55,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/5383bce47caf41b2ba0d55b9169c9998 is 50, key is test_row_0/C:col10/1732195554662/Put/seqid=0 2024-11-21T13:25:55,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742324_1500 (size=12001) 2024-11-21T13:25:55,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T13:25:55,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:55,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:25:55,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:55,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:55,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:55,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:55,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T13:25:55,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195615319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195615319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,325 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195615329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195615329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195615329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,334 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T13:25:55,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:55,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:25:55,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:55,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:55,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:55,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/5383bce47caf41b2ba0d55b9169c9998 2024-11-21T13:25:55,568 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/65c22ea15c774a6b88d3f3f516decc98 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/65c22ea15c774a6b88d3f3f516decc98 2024-11-21T13:25:55,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/65c22ea15c774a6b88d3f3f516decc98, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T13:25:55,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/8567e380ba4e41a189ce389b47cfca36 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/8567e380ba4e41a189ce389b47cfca36 2024-11-21T13:25:55,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T13:25:55,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:55,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:25:55,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:55,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:55,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:55,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/8567e380ba4e41a189ce389b47cfca36, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T13:25:55,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:55,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/5383bce47caf41b2ba0d55b9169c9998 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/5383bce47caf41b2ba0d55b9169c9998 2024-11-21T13:25:55,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/5383bce47caf41b2ba0d55b9169c9998, entries=150, sequenceid=12, filesize=11.7 K 2024-11-21T13:25:55,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8328be8165f9f405996e915498f7765f in 915ms, sequenceid=12, compaction requested=false 2024-11-21T13:25:55,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:25:55,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-21T13:25:55,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:55,726 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T13:25:55,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/99b02f101b0f409c997f5130f0000a4b is 50, key is test_row_0/A:col10/1732195554693/Put/seqid=0 2024-11-21T13:25:55,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742325_1501 (size=12001) 2024-11-21T13:25:55,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T13:25:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:55,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:25:55,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195615846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195615849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195615850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,851 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195615851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,857 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195615851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,858 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195615959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,964 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195615959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195615959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:55,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:55,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195615969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:55,973 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:56,136 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/99b02f101b0f409c997f5130f0000a4b 2024-11-21T13:25:56,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/357cf3454de248dda61c7cbd9bed9a10 is 50, key is test_row_0/B:col10/1732195554693/Put/seqid=0 2024-11-21T13:25:56,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742326_1502 (size=12001) 2024-11-21T13:25:56,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195616169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195616169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:56,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:56,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195616169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:56,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:56,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:56,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195616179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,185 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:56,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195616479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:56,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195616479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:56,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:56,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195616489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:56,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195616489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:56,550 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/357cf3454de248dda61c7cbd9bed9a10 2024-11-21T13:25:56,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/daa7bb9386dd45ed833f3747161a814b is 50, key is test_row_0/C:col10/1732195554693/Put/seqid=0 2024-11-21T13:25:56,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742327_1503 (size=12001) 2024-11-21T13:25:56,641 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T13:25:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T13:25:56,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:56,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195616859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:56,863 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_1' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:25:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:56,864 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:56,959 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/daa7bb9386dd45ed833f3747161a814b 2024-11-21T13:25:56,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/99b02f101b0f409c997f5130f0000a4b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/99b02f101b0f409c997f5130f0000a4b 2024-11-21T13:25:56,966 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/99b02f101b0f409c997f5130f0000a4b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T13:25:56,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/357cf3454de248dda61c7cbd9bed9a10 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/357cf3454de248dda61c7cbd9bed9a10 2024-11-21T13:25:56,969 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/357cf3454de248dda61c7cbd9bed9a10, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T13:25:56,969 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/daa7bb9386dd45ed833f3747161a814b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/daa7bb9386dd45ed833f3747161a814b 2024-11-21T13:25:56,972 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/daa7bb9386dd45ed833f3747161a814b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-21T13:25:56,973 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8328be8165f9f405996e915498f7765f in 1247ms, sequenceid=37, compaction requested=false 2024-11-21T13:25:56,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:25:56,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:56,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-21T13:25:56,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-21T13:25:56,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-21T13:25:56,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3190 sec 2024-11-21T13:25:56,977 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.3260 sec 2024-11-21T13:25:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:56,992 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:56,992 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T13:25:56,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/7258fb3ea5174c3dac08047fd2319fce is 50, key is test_row_0/A:col10/1732195556992/Put/seqid=0 2024-11-21T13:25:57,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742328_1504 (size=16681) 2024-11-21T13:25:57,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195617026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195617030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195617031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195617032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195617139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195617139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,154 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195617149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195617149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195617349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195617349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,355 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195617363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195617363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,401 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/7258fb3ea5174c3dac08047fd2319fce 2024-11-21T13:25:57,407 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/e01f13dc8f5442b4bf57b43849a33304 is 50, key is test_row_0/B:col10/1732195556992/Put/seqid=0 2024-11-21T13:25:57,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742329_1505 (size=12001) 2024-11-21T13:25:57,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195617659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195617659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195617679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195617679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:57,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:57,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:57,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/e01f13dc8f5442b4bf57b43849a33304 2024-11-21T13:25:57,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/2cd09f6e2f9b4594b7b19b9e0627053f is 50, key is test_row_0/C:col10/1732195556992/Put/seqid=0 2024-11-21T13:25:57,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742330_1506 (size=12001) 2024-11-21T13:25:58,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:58,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195618169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:58,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:58,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195618179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:58,185 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:58,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:58,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195618189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:58,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:58,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195618189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:58,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:58,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/2cd09f6e2f9b4594b7b19b9e0627053f 2024-11-21T13:25:58,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/7258fb3ea5174c3dac08047fd2319fce as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/7258fb3ea5174c3dac08047fd2319fce 2024-11-21T13:25:58,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/7258fb3ea5174c3dac08047fd2319fce, entries=250, sequenceid=50, filesize=16.3 K 2024-11-21T13:25:58,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/e01f13dc8f5442b4bf57b43849a33304 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/e01f13dc8f5442b4bf57b43849a33304 2024-11-21T13:25:58,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/e01f13dc8f5442b4bf57b43849a33304, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T13:25:58,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/2cd09f6e2f9b4594b7b19b9e0627053f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2cd09f6e2f9b4594b7b19b9e0627053f 2024-11-21T13:25:58,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2cd09f6e2f9b4594b7b19b9e0627053f, entries=150, sequenceid=50, filesize=11.7 K 2024-11-21T13:25:58,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8328be8165f9f405996e915498f7765f in 1239ms, sequenceid=50, compaction requested=true 2024-11-21T13:25:58,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:25:58,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:25:58,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:58,232 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:58,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:25:58,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:58,232 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:58,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:25:58,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:58,232 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:58,232 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:58,232 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/B is initiating minor compaction (all files) 2024-11-21T13:25:58,232 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/A is initiating minor compaction (all files) 2024-11-21T13:25:58,232 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/A in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:58,232 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/B in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:58,233 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/8567e380ba4e41a189ce389b47cfca36, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/357cf3454de248dda61c7cbd9bed9a10, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/e01f13dc8f5442b4bf57b43849a33304] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=35.2 K 2024-11-21T13:25:58,233 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/65c22ea15c774a6b88d3f3f516decc98, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/99b02f101b0f409c997f5130f0000a4b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/7258fb3ea5174c3dac08047fd2319fce] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=39.7 K 2024-11-21T13:25:58,233 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65c22ea15c774a6b88d3f3f516decc98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732195554659 2024-11-21T13:25:58,233 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 8567e380ba4e41a189ce389b47cfca36, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732195554659 2024-11-21T13:25:58,233 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 357cf3454de248dda61c7cbd9bed9a10, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732195554693 2024-11-21T13:25:58,233 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99b02f101b0f409c997f5130f0000a4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732195554693 2024-11-21T13:25:58,233 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7258fb3ea5174c3dac08047fd2319fce, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732195555843 2024-11-21T13:25:58,233 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e01f13dc8f5442b4bf57b43849a33304, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732195555844 2024-11-21T13:25:58,239 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#B#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:58,239 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#A#compaction#416 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:58,239 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/d67a4ad246b4452fb6616352aaee58b5 is 50, key is test_row_0/B:col10/1732195556992/Put/seqid=0 2024-11-21T13:25:58,239 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/1a40bebbd37b4a3a938231a0aa324808 is 50, key is test_row_0/A:col10/1732195556992/Put/seqid=0 2024-11-21T13:25:58,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742331_1507 (size=12104) 2024-11-21T13:25:58,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742332_1508 (size=12104) 2024-11-21T13:25:58,248 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/1a40bebbd37b4a3a938231a0aa324808 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1a40bebbd37b4a3a938231a0aa324808 2024-11-21T13:25:58,252 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/A of 8328be8165f9f405996e915498f7765f into 1a40bebbd37b4a3a938231a0aa324808(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:58,252 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:25:58,252 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/A, priority=13, startTime=1732195558232; duration=0sec 2024-11-21T13:25:58,252 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:25:58,252 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:A 2024-11-21T13:25:58,252 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:25:58,253 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:25:58,253 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/C is initiating minor compaction (all files) 2024-11-21T13:25:58,253 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/C in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:58,253 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/5383bce47caf41b2ba0d55b9169c9998, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/daa7bb9386dd45ed833f3747161a814b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2cd09f6e2f9b4594b7b19b9e0627053f] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=35.2 K 2024-11-21T13:25:58,254 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5383bce47caf41b2ba0d55b9169c9998, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1732195554659 2024-11-21T13:25:58,254 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting daa7bb9386dd45ed833f3747161a814b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732195554693 2024-11-21T13:25:58,254 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cd09f6e2f9b4594b7b19b9e0627053f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732195555844 2024-11-21T13:25:58,266 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#C#compaction#418 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:25:58,266 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/2c47c3cdd68e4c1b9208dcefb1ebeb8b is 50, key is test_row_0/C:col10/1732195556992/Put/seqid=0 2024-11-21T13:25:58,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742333_1509 (size=12104) 2024-11-21T13:25:58,282 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/2c47c3cdd68e4c1b9208dcefb1ebeb8b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2c47c3cdd68e4c1b9208dcefb1ebeb8b 2024-11-21T13:25:58,288 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/C of 8328be8165f9f405996e915498f7765f into 2c47c3cdd68e4c1b9208dcefb1ebeb8b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:58,288 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:25:58,288 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/C, priority=13, startTime=1732195558232; duration=0sec 2024-11-21T13:25:58,288 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:58,288 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:C 2024-11-21T13:25:58,646 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/d67a4ad246b4452fb6616352aaee58b5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/d67a4ad246b4452fb6616352aaee58b5 2024-11-21T13:25:58,649 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/B of 8328be8165f9f405996e915498f7765f into d67a4ad246b4452fb6616352aaee58b5(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:25:58,649 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:25:58,649 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/B, priority=13, startTime=1732195558232; duration=0sec 2024-11-21T13:25:58,649 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:25:58,649 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:B 2024-11-21T13:25:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-21T13:25:58,819 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:25:58,820 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:25:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-21T13:25:58,821 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:25:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T13:25:58,822 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:25:58,822 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:25:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:58,872 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:25:58,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T13:25:58,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/6b6badf173eb48ddab1db18fb3679e79 is 50, key is test_row_0/A:col10/1732195557031/Put/seqid=0 2024-11-21T13:25:58,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742334_1510 (size=14341) 2024-11-21T13:25:58,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/6b6badf173eb48ddab1db18fb3679e79 2024-11-21T13:25:58,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/a04c8810472c4acdbb9489fee1c169a6 is 50, key is test_row_0/B:col10/1732195557031/Put/seqid=0 2024-11-21T13:25:58,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742335_1511 (size=12001) 2024-11-21T13:25:58,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/a04c8810472c4acdbb9489fee1c169a6 2024-11-21T13:25:58,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/f807f763b65b41ba8a7c10a265553dca is 50, key is test_row_0/C:col10/1732195557031/Put/seqid=0 2024-11-21T13:25:58,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742336_1512 (size=12001) 2024-11-21T13:25:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T13:25:58,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195618941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:58,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:58,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T13:25:58,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:58,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:25:58,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:58,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:58,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:59,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:59,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195619049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:59,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,056 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:59,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T13:25:59,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:59,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:25:59,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:59,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:59,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T13:25:59,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:59,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195619179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:59,182 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:25:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:59,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:59,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195619189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:59,194 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:25:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:59,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195619199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:59,204 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:25:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:59,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195619209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:59,214 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:25:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:59,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:59,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195619259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,263 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:59,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T13:25:59,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:59,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:25:59,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:59,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:59,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:59,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:25:59,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/f807f763b65b41ba8a7c10a265553dca 2024-11-21T13:25:59,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/6b6badf173eb48ddab1db18fb3679e79 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/6b6badf173eb48ddab1db18fb3679e79 2024-11-21T13:25:59,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/6b6badf173eb48ddab1db18fb3679e79, entries=200, sequenceid=77, filesize=14.0 K 2024-11-21T13:25:59,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/a04c8810472c4acdbb9489fee1c169a6 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/a04c8810472c4acdbb9489fee1c169a6 2024-11-21T13:25:59,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/a04c8810472c4acdbb9489fee1c169a6, entries=150, sequenceid=77, filesize=11.7 K 2024-11-21T13:25:59,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/f807f763b65b41ba8a7c10a265553dca as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/f807f763b65b41ba8a7c10a265553dca 2024-11-21T13:25:59,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/f807f763b65b41ba8a7c10a265553dca, entries=150, sequenceid=77, filesize=11.7 K 2024-11-21T13:25:59,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 8328be8165f9f405996e915498f7765f in 452ms, sequenceid=77, compaction requested=false 2024-11-21T13:25:59,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:25:59,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-21T13:25:59,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:25:59,432 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T13:25:59,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/a4193ebd852041478be08c9938b0f20a is 50, key is test_row_0/A:col10/1732195558940/Put/seqid=0 2024-11-21T13:25:59,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742337_1513 (size=12001) 2024-11-21T13:25:59,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T13:25:59,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:25:59,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:25:59,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:59,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195619667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:59,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:59,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:59,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195619778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,784 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:25:59,845 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/a4193ebd852041478be08c9938b0f20a 2024-11-21T13:25:59,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/4f38522e0be84e8dbc21b11dd96535e3 is 50, key is test_row_0/B:col10/1732195558940/Put/seqid=0 2024-11-21T13:25:59,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742338_1514 (size=12001) 2024-11-21T13:25:59,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T13:25:59,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:25:59,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195619989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:25:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:25:59,994 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:00,295 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/4f38522e0be84e8dbc21b11dd96535e3 2024-11-21T13:26:00,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/4cb9a36aa1e54d2f823b5b5683559691 is 50, key is test_row_0/C:col10/1732195558940/Put/seqid=0 2024-11-21T13:26:00,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742339_1515 (size=12001) 2024-11-21T13:26:00,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:00,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195620299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:00,306 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:00,308 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/4cb9a36aa1e54d2f823b5b5683559691 2024-11-21T13:26:00,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/a4193ebd852041478be08c9938b0f20a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/a4193ebd852041478be08c9938b0f20a 2024-11-21T13:26:00,314 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/a4193ebd852041478be08c9938b0f20a, entries=150, sequenceid=89, filesize=11.7 K 2024-11-21T13:26:00,315 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/4f38522e0be84e8dbc21b11dd96535e3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4f38522e0be84e8dbc21b11dd96535e3 2024-11-21T13:26:00,318 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4f38522e0be84e8dbc21b11dd96535e3, entries=150, sequenceid=89, filesize=11.7 K 2024-11-21T13:26:00,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/4cb9a36aa1e54d2f823b5b5683559691 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/4cb9a36aa1e54d2f823b5b5683559691 2024-11-21T13:26:00,321 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/4cb9a36aa1e54d2f823b5b5683559691, entries=150, sequenceid=89, filesize=11.7 K 2024-11-21T13:26:00,322 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 8328be8165f9f405996e915498f7765f in 890ms, sequenceid=89, compaction requested=true 2024-11-21T13:26:00,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:00,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:00,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-21T13:26:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-21T13:26:00,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-21T13:26:00,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5020 sec 2024-11-21T13:26:00,327 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.5060 sec 2024-11-21T13:26:00,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:00,812 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:00,812 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T13:26:00,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/0e7b2526d76c4556a9eee03ce8c49ae9 is 50, key is test_row_0/A:col10/1732195559658/Put/seqid=0 2024-11-21T13:26:00,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742340_1516 (size=14341) 2024-11-21T13:26:00,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:00,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195620852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:00,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:00,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:00,858 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:00,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-21T13:26:00,970 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:00,970 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-21T13:26:00,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T13:26:00,972 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:00,972 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:00,972 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:00,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:00,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195620969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:00,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:01,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T13:26:01,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T13:26:01,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:01,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:01,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:01,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:01,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:01,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:01,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:01,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195621189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:01,195 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:01,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:01,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195621199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:01,203 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:01,203 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:01,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:01,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195621209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:01,215 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:01,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:01,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:01,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:01,219 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/0e7b2526d76c4556a9eee03ce8c49ae9 2024-11-21T13:26:01,226 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/d79a89bfe63546e0b156f8d4f1daa2b0 is 50, key is test_row_0/B:col10/1732195559658/Put/seqid=0 2024-11-21T13:26:01,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742341_1517 (size=12001) 2024-11-21T13:26:01,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195621229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:01,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:01,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195621229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:01,232 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:01,233 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:01,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:01,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T13:26:01,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:01,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:01,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:01,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:01,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:01,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:01,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T13:26:01,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T13:26:01,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:01,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:01,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:01,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:01,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:01,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:01,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:01,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195621499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:01,505 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:01,582 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T13:26:01,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:01,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:01,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:01,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:01,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:01,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T13:26:01,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/d79a89bfe63546e0b156f8d4f1daa2b0 2024-11-21T13:26:01,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/421d3b6c46734bd18b7fd363942d2f9f is 50, key is test_row_0/C:col10/1732195559658/Put/seqid=0 2024-11-21T13:26:01,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742342_1518 (size=12001) 2024-11-21T13:26:01,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/421d3b6c46734bd18b7fd363942d2f9f 2024-11-21T13:26:01,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/0e7b2526d76c4556a9eee03ce8c49ae9 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/0e7b2526d76c4556a9eee03ce8c49ae9 2024-11-21T13:26:01,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/0e7b2526d76c4556a9eee03ce8c49ae9, entries=200, sequenceid=114, filesize=14.0 K 2024-11-21T13:26:01,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/d79a89bfe63546e0b156f8d4f1daa2b0 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/d79a89bfe63546e0b156f8d4f1daa2b0 2024-11-21T13:26:01,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/d79a89bfe63546e0b156f8d4f1daa2b0, entries=150, sequenceid=114, filesize=11.7 K 2024-11-21T13:26:01,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/421d3b6c46734bd18b7fd363942d2f9f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/421d3b6c46734bd18b7fd363942d2f9f 2024-11-21T13:26:01,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/421d3b6c46734bd18b7fd363942d2f9f, entries=150, sequenceid=114, filesize=11.7 K 2024-11-21T13:26:01,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8328be8165f9f405996e915498f7765f in 839ms, sequenceid=114, compaction requested=true 2024-11-21T13:26:01,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:01,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:01,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:01,652 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:26:01,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:01,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:01,652 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:26:01,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:01,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:01,653 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52787 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:26:01,653 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:26:01,653 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/A is initiating minor compaction (all files) 2024-11-21T13:26:01,653 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/B is initiating minor compaction (all files) 2024-11-21T13:26:01,653 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/A in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:01,653 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/B in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:01,653 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1a40bebbd37b4a3a938231a0aa324808, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/6b6badf173eb48ddab1db18fb3679e79, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/a4193ebd852041478be08c9938b0f20a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/0e7b2526d76c4556a9eee03ce8c49ae9] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=51.5 K 2024-11-21T13:26:01,653 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/d67a4ad246b4452fb6616352aaee58b5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/a04c8810472c4acdbb9489fee1c169a6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4f38522e0be84e8dbc21b11dd96535e3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/d79a89bfe63546e0b156f8d4f1daa2b0] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=47.0 K 2024-11-21T13:26:01,653 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a40bebbd37b4a3a938231a0aa324808, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732195555844 2024-11-21T13:26:01,653 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting d67a4ad246b4452fb6616352aaee58b5, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732195555844 2024-11-21T13:26:01,653 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b6badf173eb48ddab1db18fb3679e79, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732195557021 2024-11-21T13:26:01,653 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting a04c8810472c4acdbb9489fee1c169a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732195557021 2024-11-21T13:26:01,654 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4193ebd852041478be08c9938b0f20a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732195558908 2024-11-21T13:26:01,654 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f38522e0be84e8dbc21b11dd96535e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732195558908 2024-11-21T13:26:01,654 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e7b2526d76c4556a9eee03ce8c49ae9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732195559639 2024-11-21T13:26:01,654 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting d79a89bfe63546e0b156f8d4f1daa2b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732195559658 2024-11-21T13:26:01,661 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#B#compaction#428 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:01,661 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#A#compaction#429 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:01,661 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/66b42b10c79e460a93d4f7cf77d5c7be is 50, key is test_row_0/A:col10/1732195559658/Put/seqid=0 2024-11-21T13:26:01,661 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/8c8bd421be794a36a3289cfce011d1e7 is 50, key is test_row_0/B:col10/1732195559658/Put/seqid=0 2024-11-21T13:26:01,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742343_1519 (size=12241) 2024-11-21T13:26:01,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742344_1520 (size=12241) 2024-11-21T13:26:01,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-21T13:26:01,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:01,735 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:26:01,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/e63447d0fa8147e58a1b6c603b02e382 is 50, key is test_row_0/A:col10/1732195560829/Put/seqid=0 2024-11-21T13:26:01,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742345_1521 (size=12001) 2024-11-21T13:26:02,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:02,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:02,067 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/8c8bd421be794a36a3289cfce011d1e7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/8c8bd421be794a36a3289cfce011d1e7 2024-11-21T13:26:02,070 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/66b42b10c79e460a93d4f7cf77d5c7be as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/66b42b10c79e460a93d4f7cf77d5c7be 2024-11-21T13:26:02,071 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8328be8165f9f405996e915498f7765f/B of 8328be8165f9f405996e915498f7765f into 8c8bd421be794a36a3289cfce011d1e7(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:02,071 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:02,071 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/B, priority=12, startTime=1732195561652; duration=0sec 2024-11-21T13:26:02,072 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:02,072 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:B 2024-11-21T13:26:02,072 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:26:02,073 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:26:02,073 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/C is initiating minor compaction (all files) 2024-11-21T13:26:02,073 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/C in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:02,073 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2c47c3cdd68e4c1b9208dcefb1ebeb8b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/f807f763b65b41ba8a7c10a265553dca, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/4cb9a36aa1e54d2f823b5b5683559691, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/421d3b6c46734bd18b7fd363942d2f9f] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=47.0 K 2024-11-21T13:26:02,073 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8328be8165f9f405996e915498f7765f/A of 8328be8165f9f405996e915498f7765f into 66b42b10c79e460a93d4f7cf77d5c7be(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:02,073 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:02,073 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/A, priority=12, startTime=1732195561651; duration=0sec 2024-11-21T13:26:02,073 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:02,073 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:A 2024-11-21T13:26:02,073 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c47c3cdd68e4c1b9208dcefb1ebeb8b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732195555844 2024-11-21T13:26:02,073 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f807f763b65b41ba8a7c10a265553dca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732195557021 2024-11-21T13:26:02,074 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cb9a36aa1e54d2f823b5b5683559691, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732195558908 2024-11-21T13:26:02,074 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 421d3b6c46734bd18b7fd363942d2f9f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732195559658 2024-11-21T13:26:02,080 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:02,080 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/2539795f06a84bc3a8723ea81f4e502d is 50, key is test_row_0/C:col10/1732195559658/Put/seqid=0 2024-11-21T13:26:02,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742346_1522 (size=12241) 2024-11-21T13:26:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T13:26:02,142 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/e63447d0fa8147e58a1b6c603b02e382 2024-11-21T13:26:02,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/fa7f2cdcf50a464cb7909cd84719e45c is 50, key is test_row_0/B:col10/1732195560829/Put/seqid=0 2024-11-21T13:26:02,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195622151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:02,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:02,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742347_1523 (size=12001) 2024-11-21T13:26:02,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195622259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:02,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:02,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195622469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:02,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:02,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:02,475 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:02,487 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/2539795f06a84bc3a8723ea81f4e502d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2539795f06a84bc3a8723ea81f4e502d 2024-11-21T13:26:02,490 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8328be8165f9f405996e915498f7765f/C of 8328be8165f9f405996e915498f7765f into 2539795f06a84bc3a8723ea81f4e502d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:02,490 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:02,490 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/C, priority=12, startTime=1732195561652; duration=0sec 2024-11-21T13:26:02,490 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:02,490 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:C 2024-11-21T13:26:02,557 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/fa7f2cdcf50a464cb7909cd84719e45c 2024-11-21T13:26:02,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/848dee9956934d4396d0f465f44bbba5 is 50, key is test_row_0/C:col10/1732195560829/Put/seqid=0 2024-11-21T13:26:02,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742348_1524 (size=12001) 2024-11-21T13:26:02,565 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/848dee9956934d4396d0f465f44bbba5 2024-11-21T13:26:02,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/e63447d0fa8147e58a1b6c603b02e382 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/e63447d0fa8147e58a1b6c603b02e382 2024-11-21T13:26:02,571 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/e63447d0fa8147e58a1b6c603b02e382, entries=150, sequenceid=125, filesize=11.7 K 2024-11-21T13:26:02,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/fa7f2cdcf50a464cb7909cd84719e45c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fa7f2cdcf50a464cb7909cd84719e45c 2024-11-21T13:26:02,574 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fa7f2cdcf50a464cb7909cd84719e45c, entries=150, sequenceid=125, filesize=11.7 K 2024-11-21T13:26:02,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/848dee9956934d4396d0f465f44bbba5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/848dee9956934d4396d0f465f44bbba5 2024-11-21T13:26:02,578 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/848dee9956934d4396d0f465f44bbba5, entries=150, sequenceid=125, filesize=11.7 K 2024-11-21T13:26:02,579 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8328be8165f9f405996e915498f7765f in 845ms, sequenceid=125, compaction requested=false 2024-11-21T13:26:02,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:02,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:02,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-21T13:26:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-21T13:26:02,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-21T13:26:02,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6080 sec 2024-11-21T13:26:02,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.6130 sec 2024-11-21T13:26:02,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:02,783 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:02,783 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-21T13:26:02,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/89d088e0085b44e890b07ce40c8f3360 is 50, key is test_row_0/A:col10/1732195562782/Put/seqid=0 2024-11-21T13:26:02,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742349_1525 (size=14541) 2024-11-21T13:26:02,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/89d088e0085b44e890b07ce40c8f3360 2024-11-21T13:26:02,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/95cdf4e1032443129384d51a0d7dfb3c is 50, key is test_row_0/B:col10/1732195562782/Put/seqid=0 2024-11-21T13:26:02,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742350_1526 (size=12151) 2024-11-21T13:26:02,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:02,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195622824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:02,832 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:02,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:02,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195622939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:02,943 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:03,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-21T13:26:03,120 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:03,120 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-21T13:26:03,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T13:26:03,122 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:03,122 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:03,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:03,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:03,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195623149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:03,154 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:03,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/95cdf4e1032443129384d51a0d7dfb3c 2024-11-21T13:26:03,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/127edb3550a842c7b83e6c26718205a6 is 50, key is test_row_0/C:col10/1732195562782/Put/seqid=0 2024-11-21T13:26:03,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742351_1527 (size=12151) 2024-11-21T13:26:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T13:26:03,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T13:26:03,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:03,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:03,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:03,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:03,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:03,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:03,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T13:26:03,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:03,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:03,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:03,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:03,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T13:26:03,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:03,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195623459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:03,462 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:03,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T13:26:03,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:03,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:03,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:03,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:03,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:03,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:03,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/127edb3550a842c7b83e6c26718205a6 2024-11-21T13:26:03,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/89d088e0085b44e890b07ce40c8f3360 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/89d088e0085b44e890b07ce40c8f3360 2024-11-21T13:26:03,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/89d088e0085b44e890b07ce40c8f3360, entries=200, sequenceid=154, filesize=14.2 K 2024-11-21T13:26:03,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/95cdf4e1032443129384d51a0d7dfb3c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/95cdf4e1032443129384d51a0d7dfb3c 2024-11-21T13:26:03,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/95cdf4e1032443129384d51a0d7dfb3c, entries=150, sequenceid=154, filesize=11.9 K 2024-11-21T13:26:03,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/127edb3550a842c7b83e6c26718205a6 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/127edb3550a842c7b83e6c26718205a6 2024-11-21T13:26:03,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/127edb3550a842c7b83e6c26718205a6, entries=150, sequenceid=154, filesize=11.9 K 2024-11-21T13:26:03,621 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 8328be8165f9f405996e915498f7765f in 838ms, sequenceid=154, compaction requested=true 2024-11-21T13:26:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:03,621 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:03,621 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:03,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:03,622 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:03,622 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:03,622 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/A is initiating minor compaction (all files) 2024-11-21T13:26:03,622 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/A in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:03,622 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:03,622 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/B is initiating minor compaction (all files) 2024-11-21T13:26:03,622 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/66b42b10c79e460a93d4f7cf77d5c7be, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/e63447d0fa8147e58a1b6c603b02e382, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/89d088e0085b44e890b07ce40c8f3360] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=37.9 K 2024-11-21T13:26:03,622 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/B in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:03,622 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/8c8bd421be794a36a3289cfce011d1e7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fa7f2cdcf50a464cb7909cd84719e45c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/95cdf4e1032443129384d51a0d7dfb3c] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=35.5 K 2024-11-21T13:26:03,622 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66b42b10c79e460a93d4f7cf77d5c7be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732195559658 2024-11-21T13:26:03,623 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c8bd421be794a36a3289cfce011d1e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732195559658 2024-11-21T13:26:03,623 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting e63447d0fa8147e58a1b6c603b02e382, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732195560829 2024-11-21T13:26:03,623 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting fa7f2cdcf50a464cb7909cd84719e45c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732195560829 2024-11-21T13:26:03,623 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89d088e0085b44e890b07ce40c8f3360, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732195562130 2024-11-21T13:26:03,623 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 95cdf4e1032443129384d51a0d7dfb3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732195562139 2024-11-21T13:26:03,629 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#B#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:03,629 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/fd185279849841948cdf326f14048485 is 50, key is test_row_0/B:col10/1732195562782/Put/seqid=0 2024-11-21T13:26:03,630 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#A#compaction#438 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:03,630 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/c6175a9b081d47789979894ebbf0d362 is 50, key is test_row_0/A:col10/1732195562782/Put/seqid=0 2024-11-21T13:26:03,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742352_1528 (size=12493) 2024-11-21T13:26:03,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742353_1529 (size=12493) 2024-11-21T13:26:03,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-21T13:26:03,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:03,739 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-21T13:26:03,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/0b037d9de7584e818dc645717e4e0231 is 50, key is test_row_0/A:col10/1732195562823/Put/seqid=0 2024-11-21T13:26:03,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742354_1530 (size=12151) 2024-11-21T13:26:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T13:26:03,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:03,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:04,042 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/fd185279849841948cdf326f14048485 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fd185279849841948cdf326f14048485 2024-11-21T13:26:04,045 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/c6175a9b081d47789979894ebbf0d362 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/c6175a9b081d47789979894ebbf0d362 2024-11-21T13:26:04,046 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/B of 8328be8165f9f405996e915498f7765f into fd185279849841948cdf326f14048485(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:04,046 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:04,046 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/B, priority=13, startTime=1732195563621; duration=0sec 2024-11-21T13:26:04,047 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:04,047 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:B 2024-11-21T13:26:04,047 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:04,048 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:04,048 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/C is initiating minor compaction (all files) 2024-11-21T13:26:04,048 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/C in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:04,048 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2539795f06a84bc3a8723ea81f4e502d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/848dee9956934d4396d0f465f44bbba5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/127edb3550a842c7b83e6c26718205a6] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=35.5 K 2024-11-21T13:26:04,048 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 2539795f06a84bc3a8723ea81f4e502d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732195559658 2024-11-21T13:26:04,048 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 848dee9956934d4396d0f465f44bbba5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1732195560829 2024-11-21T13:26:04,049 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/A of 8328be8165f9f405996e915498f7765f into c6175a9b081d47789979894ebbf0d362(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:04,049 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:04,049 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/A, priority=13, startTime=1732195563621; duration=0sec 2024-11-21T13:26:04,049 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:04,049 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:A 2024-11-21T13:26:04,049 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 127edb3550a842c7b83e6c26718205a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732195562139 2024-11-21T13:26:04,055 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#C#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:04,056 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/10242f7abe1f48749c6662318db5a13c is 50, key is test_row_0/C:col10/1732195562782/Put/seqid=0 2024-11-21T13:26:04,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742355_1531 (size=12493) 2024-11-21T13:26:04,071 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/10242f7abe1f48749c6662318db5a13c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/10242f7abe1f48749c6662318db5a13c 2024-11-21T13:26:04,076 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/C of 8328be8165f9f405996e915498f7765f into 10242f7abe1f48749c6662318db5a13c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:04,076 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:04,076 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/C, priority=13, startTime=1732195563621; duration=0sec 2024-11-21T13:26:04,076 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:04,076 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:C 2024-11-21T13:26:04,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:04,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195624109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:04,118 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:04,145 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/0b037d9de7584e818dc645717e4e0231 2024-11-21T13:26:04,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/893f9c8219c94ec79953a23a1f5dd842 is 50, key is test_row_0/B:col10/1732195562823/Put/seqid=0 2024-11-21T13:26:04,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742356_1532 (size=12151) 2024-11-21T13:26:04,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:04,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195624229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:04,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:04,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:04,230 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T13:26:04,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:04,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195624439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:04,443 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:04,553 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/893f9c8219c94ec79953a23a1f5dd842 2024-11-21T13:26:04,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/79522ebb42034b828e51c746e9ce3510 is 50, key is test_row_0/C:col10/1732195562823/Put/seqid=0 2024-11-21T13:26:04,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742357_1533 (size=12151) 2024-11-21T13:26:04,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:04,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195624749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:04,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:04,754 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:04,755 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:04,964 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/79522ebb42034b828e51c746e9ce3510 2024-11-21T13:26:05,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/0b037d9de7584e818dc645717e4e0231 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/0b037d9de7584e818dc645717e4e0231 2024-11-21T13:26:05,018 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/0b037d9de7584e818dc645717e4e0231, entries=150, sequenceid=164, filesize=11.9 K 2024-11-21T13:26:05,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/893f9c8219c94ec79953a23a1f5dd842 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/893f9c8219c94ec79953a23a1f5dd842 2024-11-21T13:26:05,021 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/893f9c8219c94ec79953a23a1f5dd842, entries=150, sequenceid=164, filesize=11.9 K 2024-11-21T13:26:05,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/79522ebb42034b828e51c746e9ce3510 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/79522ebb42034b828e51c746e9ce3510 2024-11-21T13:26:05,024 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/79522ebb42034b828e51c746e9ce3510, entries=150, sequenceid=164, filesize=11.9 K 2024-11-21T13:26:05,025 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 8328be8165f9f405996e915498f7765f in 1287ms, sequenceid=164, compaction requested=false 2024-11-21T13:26:05,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:05,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:05,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-21T13:26:05,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-21T13:26:05,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-21T13:26:05,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9040 sec 2024-11-21T13:26:05,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.9080 sec 2024-11-21T13:26:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:05,223 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:05,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-21T13:26:05,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/d5494c0bec4446a490b1ae360271370b is 50, key is test_row_0/A:col10/1732195565222/Put/seqid=0 2024-11-21T13:26:05,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742358_1534 (size=14541) 2024-11-21T13:26:05,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:05,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195625252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:05,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,256 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:05,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195625259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:05,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:05,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195625259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:05,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:05,265 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:05,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195625259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,266 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,266 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:05,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-21T13:26:05,269 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:05,270 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-21T13:26:05,271 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:05,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T13:26:05,272 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:05,272 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:05,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:05,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195625269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:05,277 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,278 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:05,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:05,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195625359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T13:26:05,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T13:26:05,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:05,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:05,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:05,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:05,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:05,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:05,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T13:26:05,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:05,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:05,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:05,577 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:05,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:05,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:05,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:05,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195625579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:05,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,583 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:05,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T13:26:05,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/d5494c0bec4446a490b1ae360271370b 2024-11-21T13:26:05,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/4a744a950fda4b7b9e0984dff71ca29d is 50, key is test_row_0/B:col10/1732195565222/Put/seqid=0 2024-11-21T13:26:05,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742359_1535 (size=12151) 2024-11-21T13:26:05,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T13:26:05,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:05,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:05,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:05,730 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:05,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:05,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:05,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T13:26:05,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:05,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:05,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:05,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:05,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:05,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:05,894 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:05,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195625889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:05,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:05,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T13:26:06,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T13:26:06,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:06,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:06,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:06,035 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:06,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:06,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:06,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/4a744a950fda4b7b9e0984dff71ca29d 2024-11-21T13:26:06,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/ee5893e40c264b79b3a95d02fda31e4d is 50, key is test_row_0/C:col10/1732195565222/Put/seqid=0 2024-11-21T13:26:06,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742360_1536 (size=12151) 2024-11-21T13:26:06,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T13:26:06,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:06,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:06,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:06,188 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:06,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:06,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:06,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:06,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195626279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:06,283 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2174 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:06,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:06,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:06,284 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:06,340 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T13:26:06,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:06,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:06,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:06,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:06,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:06,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:06,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:06,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195626399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:06,403 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:06,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:06,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T13:26:06,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/ee5893e40c264b79b3a95d02fda31e4d 2024-11-21T13:26:06,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/d5494c0bec4446a490b1ae360271370b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/d5494c0bec4446a490b1ae360271370b 2024-11-21T13:26:06,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/d5494c0bec4446a490b1ae360271370b, entries=200, sequenceid=194, filesize=14.2 K 2024-11-21T13:26:06,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/4a744a950fda4b7b9e0984dff71ca29d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4a744a950fda4b7b9e0984dff71ca29d 2024-11-21T13:26:06,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4a744a950fda4b7b9e0984dff71ca29d, entries=150, sequenceid=194, filesize=11.9 K 2024-11-21T13:26:06,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/ee5893e40c264b79b3a95d02fda31e4d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/ee5893e40c264b79b3a95d02fda31e4d 2024-11-21T13:26:06,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/ee5893e40c264b79b3a95d02fda31e4d, entries=150, sequenceid=194, filesize=11.9 K 2024-11-21T13:26:06,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 8328be8165f9f405996e915498f7765f in 1241ms, sequenceid=194, compaction requested=true 2024-11-21T13:26:06,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:06,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:06,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:06,465 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:06,465 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:06,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:06,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:06,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:06,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:06,465 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:06,465 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:06,465 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/A is initiating minor compaction (all files) 2024-11-21T13:26:06,465 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/B is initiating minor compaction (all files) 2024-11-21T13:26:06,465 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/B in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:06,465 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/A in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:06,466 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fd185279849841948cdf326f14048485, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/893f9c8219c94ec79953a23a1f5dd842, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4a744a950fda4b7b9e0984dff71ca29d] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=35.9 K 2024-11-21T13:26:06,466 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/c6175a9b081d47789979894ebbf0d362, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/0b037d9de7584e818dc645717e4e0231, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/d5494c0bec4446a490b1ae360271370b] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=38.3 K 2024-11-21T13:26:06,466 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6175a9b081d47789979894ebbf0d362, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732195562139 2024-11-21T13:26:06,466 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting fd185279849841948cdf326f14048485, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732195562139 2024-11-21T13:26:06,466 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b037d9de7584e818dc645717e4e0231, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732195562809 2024-11-21T13:26:06,466 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 893f9c8219c94ec79953a23a1f5dd842, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732195562809 2024-11-21T13:26:06,466 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5494c0bec4446a490b1ae360271370b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732195564087 2024-11-21T13:26:06,467 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a744a950fda4b7b9e0984dff71ca29d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732195564087 2024-11-21T13:26:06,473 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#B#compaction#446 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:06,473 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/443c8d3f72d343b98106ec6ff6e41822 is 50, key is test_row_0/B:col10/1732195565222/Put/seqid=0 2024-11-21T13:26:06,486 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#A#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:06,487 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/1cf40fb34cf34e2a9de8b96aecae96d7 is 50, key is test_row_0/A:col10/1732195565222/Put/seqid=0 2024-11-21T13:26:06,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742361_1537 (size=12595) 2024-11-21T13:26:06,492 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/443c8d3f72d343b98106ec6ff6e41822 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/443c8d3f72d343b98106ec6ff6e41822 2024-11-21T13:26:06,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-21T13:26:06,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:06,495 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-21T13:26:06,502 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/B of 8328be8165f9f405996e915498f7765f into 443c8d3f72d343b98106ec6ff6e41822(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:06,502 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:06,502 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/B, priority=13, startTime=1732195566465; duration=0sec 2024-11-21T13:26:06,502 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:06,502 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:B 2024-11-21T13:26:06,502 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:06,503 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:06,503 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/C is initiating minor compaction (all files) 2024-11-21T13:26:06,503 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/C in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:06,503 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/10242f7abe1f48749c6662318db5a13c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/79522ebb42034b828e51c746e9ce3510, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/ee5893e40c264b79b3a95d02fda31e4d] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=35.9 K 2024-11-21T13:26:06,503 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 10242f7abe1f48749c6662318db5a13c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732195562139 2024-11-21T13:26:06,504 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 79522ebb42034b828e51c746e9ce3510, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1732195562809 2024-11-21T13:26:06,504 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting ee5893e40c264b79b3a95d02fda31e4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732195564087 2024-11-21T13:26:06,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742362_1538 (size=12595) 2024-11-21T13:26:06,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/76620f5ea584460f98ff894717fa877f is 50, key is test_row_0/A:col10/1732195565250/Put/seqid=0 2024-11-21T13:26:06,511 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#C#compaction#449 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:06,511 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/5d970b6a099742d491d58a770659cdda is 50, key is test_row_0/C:col10/1732195565222/Put/seqid=0 2024-11-21T13:26:06,518 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/1cf40fb34cf34e2a9de8b96aecae96d7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1cf40fb34cf34e2a9de8b96aecae96d7 2024-11-21T13:26:06,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742364_1540 (size=12595) 2024-11-21T13:26:06,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742363_1539 (size=12151) 2024-11-21T13:26:06,523 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/A of 8328be8165f9f405996e915498f7765f into 1cf40fb34cf34e2a9de8b96aecae96d7(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:06,523 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:06,523 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/A, priority=13, startTime=1732195566464; duration=0sec 2024-11-21T13:26:06,523 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:06,523 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:A 2024-11-21T13:26:06,524 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/76620f5ea584460f98ff894717fa877f 2024-11-21T13:26:06,529 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/5d970b6a099742d491d58a770659cdda as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/5d970b6a099742d491d58a770659cdda 2024-11-21T13:26:06,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/af7c46cb9f1b4d16a63ca539c8ba5af8 is 50, key is test_row_0/B:col10/1732195565250/Put/seqid=0 2024-11-21T13:26:06,534 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/C of 8328be8165f9f405996e915498f7765f into 5d970b6a099742d491d58a770659cdda(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:06,534 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:06,534 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/C, priority=13, startTime=1732195566465; duration=0sec 2024-11-21T13:26:06,534 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:06,534 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:C 2024-11-21T13:26:06,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742365_1541 (size=12151) 2024-11-21T13:26:06,937 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/af7c46cb9f1b4d16a63ca539c8ba5af8 2024-11-21T13:26:06,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/da5b70827dd34d08ac9975019934d214 is 50, key is test_row_0/C:col10/1732195565250/Put/seqid=0 2024-11-21T13:26:06,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742366_1542 (size=12151) 2024-11-21T13:26:06,956 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/da5b70827dd34d08ac9975019934d214 2024-11-21T13:26:06,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/76620f5ea584460f98ff894717fa877f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/76620f5ea584460f98ff894717fa877f 2024-11-21T13:26:06,964 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/76620f5ea584460f98ff894717fa877f, entries=150, sequenceid=203, filesize=11.9 K 2024-11-21T13:26:06,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/af7c46cb9f1b4d16a63ca539c8ba5af8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/af7c46cb9f1b4d16a63ca539c8ba5af8 2024-11-21T13:26:06,969 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/af7c46cb9f1b4d16a63ca539c8ba5af8, entries=150, sequenceid=203, filesize=11.9 K 2024-11-21T13:26:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/da5b70827dd34d08ac9975019934d214 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/da5b70827dd34d08ac9975019934d214 2024-11-21T13:26:06,974 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/da5b70827dd34d08ac9975019934d214, entries=150, sequenceid=203, filesize=11.9 K 2024-11-21T13:26:06,975 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 8328be8165f9f405996e915498f7765f in 480ms, sequenceid=203, compaction requested=false 2024-11-21T13:26:06,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:06,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:06,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-21T13:26:06,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-21T13:26:06,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-21T13:26:06,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7050 sec 2024-11-21T13:26:06,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.7100 sec 2024-11-21T13:26:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-21T13:26:07,420 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:07,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-21T13:26:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T13:26:07,422 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:07,423 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:07,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:07,459 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:07,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:26:07,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/d6c42c1b22f94bdb927cf1f8fdce1a03 is 50, key is test_row_0/A:col10/1732195567458/Put/seqid=0 2024-11-21T13:26:07,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742367_1543 (size=12151) 2024-11-21T13:26:07,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/d6c42c1b22f94bdb927cf1f8fdce1a03 2024-11-21T13:26:07,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/1a33aa2ae11b4987a4e5cc57f068f77f is 50, key is test_row_0/B:col10/1732195567458/Put/seqid=0 2024-11-21T13:26:07,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742368_1544 (size=12151) 2024-11-21T13:26:07,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/1a33aa2ae11b4987a4e5cc57f068f77f 2024-11-21T13:26:07,493 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/59afa3323c464903a8bfd00f62576b05 is 50, key is test_row_0/C:col10/1732195567458/Put/seqid=0 2024-11-21T13:26:07,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742369_1545 (size=12151) 2024-11-21T13:26:07,498 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=217 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/59afa3323c464903a8bfd00f62576b05 2024-11-21T13:26:07,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/d6c42c1b22f94bdb927cf1f8fdce1a03 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/d6c42c1b22f94bdb927cf1f8fdce1a03 2024-11-21T13:26:07,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/d6c42c1b22f94bdb927cf1f8fdce1a03, entries=150, sequenceid=217, filesize=11.9 K 2024-11-21T13:26:07,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/1a33aa2ae11b4987a4e5cc57f068f77f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/1a33aa2ae11b4987a4e5cc57f068f77f 2024-11-21T13:26:07,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/1a33aa2ae11b4987a4e5cc57f068f77f, entries=150, sequenceid=217, filesize=11.9 K 2024-11-21T13:26:07,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/59afa3323c464903a8bfd00f62576b05 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/59afa3323c464903a8bfd00f62576b05 2024-11-21T13:26:07,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/59afa3323c464903a8bfd00f62576b05, entries=150, sequenceid=217, filesize=11.9 K 2024-11-21T13:26:07,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=46.96 KB/48090 for 8328be8165f9f405996e915498f7765f in 64ms, sequenceid=217, compaction requested=true 2024-11-21T13:26:07,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:07,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:07,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:07,523 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:07,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:07,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:07,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:07,523 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:07,523 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:07,523 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:07,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:26:07,525 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:07,525 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/A is initiating minor compaction (all files) 2024-11-21T13:26:07,525 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/A in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:07,525 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1cf40fb34cf34e2a9de8b96aecae96d7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/76620f5ea584460f98ff894717fa877f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/d6c42c1b22f94bdb927cf1f8fdce1a03] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=36.0 K 2024-11-21T13:26:07,525 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:07,525 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/B is initiating minor compaction (all files) 2024-11-21T13:26:07,525 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/B in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:07,526 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/443c8d3f72d343b98106ec6ff6e41822, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/af7c46cb9f1b4d16a63ca539c8ba5af8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/1a33aa2ae11b4987a4e5cc57f068f77f] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=36.0 K 2024-11-21T13:26:07,526 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1cf40fb34cf34e2a9de8b96aecae96d7, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732195564087 2024-11-21T13:26:07,526 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 443c8d3f72d343b98106ec6ff6e41822, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732195564087 2024-11-21T13:26:07,526 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76620f5ea584460f98ff894717fa877f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732195565235 2024-11-21T13:26:07,526 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting af7c46cb9f1b4d16a63ca539c8ba5af8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732195565235 2024-11-21T13:26:07,527 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6c42c1b22f94bdb927cf1f8fdce1a03, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732195567433 2024-11-21T13:26:07,527 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a33aa2ae11b4987a4e5cc57f068f77f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732195567433 2024-11-21T13:26:07,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/f7aa9bad3f0346129de8dc5d2c34ad28 is 50, key is test_row_0/A:col10/1732195567518/Put/seqid=0 2024-11-21T13:26:07,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T13:26:07,546 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#A#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:07,546 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#B#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:07,547 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/fddd0311fa784f908150349dc7432d86 is 50, key is test_row_0/B:col10/1732195567458/Put/seqid=0 2024-11-21T13:26:07,547 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/2f863e9d891e44468e5e20a11e4c07f2 is 50, key is test_row_0/A:col10/1732195567458/Put/seqid=0 2024-11-21T13:26:07,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742370_1546 (size=14541) 2024-11-21T13:26:07,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/f7aa9bad3f0346129de8dc5d2c34ad28 2024-11-21T13:26:07,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742371_1547 (size=12697) 2024-11-21T13:26:07,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742372_1548 (size=12697) 2024-11-21T13:26:07,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/918dcf9e0934403cb7dee3459763a3ca is 50, key is test_row_0/B:col10/1732195567518/Put/seqid=0 2024-11-21T13:26:07,571 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/2f863e9d891e44468e5e20a11e4c07f2 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/2f863e9d891e44468e5e20a11e4c07f2 2024-11-21T13:26:07,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742373_1549 (size=12151) 2024-11-21T13:26:07,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T13:26:07,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:07,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:07,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:07,576 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/fddd0311fa784f908150349dc7432d86 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fddd0311fa784f908150349dc7432d86 2024-11-21T13:26:07,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:07,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:07,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/918dcf9e0934403cb7dee3459763a3ca 2024-11-21T13:26:07,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:07,580 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/A of 8328be8165f9f405996e915498f7765f into 2f863e9d891e44468e5e20a11e4c07f2(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:07,580 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:07,581 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/A, priority=13, startTime=1732195567523; duration=0sec 2024-11-21T13:26:07,581 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:07,581 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:A 2024-11-21T13:26:07,581 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:07,585 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/B of 8328be8165f9f405996e915498f7765f into fddd0311fa784f908150349dc7432d86(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:07,585 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:07,585 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/B, priority=13, startTime=1732195567523; duration=0sec 2024-11-21T13:26:07,585 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:07,585 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:B 2024-11-21T13:26:07,586 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:07,586 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/C is initiating minor compaction (all files) 2024-11-21T13:26:07,586 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/C in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:07,586 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/5d970b6a099742d491d58a770659cdda, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/da5b70827dd34d08ac9975019934d214, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/59afa3323c464903a8bfd00f62576b05] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=36.0 K 2024-11-21T13:26:07,587 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d970b6a099742d491d58a770659cdda, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732195564087 2024-11-21T13:26:07,588 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting da5b70827dd34d08ac9975019934d214, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1732195565235 2024-11-21T13:26:07,589 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59afa3323c464903a8bfd00f62576b05, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732195567433 2024-11-21T13:26:07,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/85a093eff63547158a2116d1b659dd04 is 50, key is test_row_0/C:col10/1732195567518/Put/seqid=0 2024-11-21T13:26:07,596 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#C#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:07,596 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/f3f524a5c4f24c4d88ae919bad3a908f is 50, key is test_row_0/C:col10/1732195567458/Put/seqid=0 2024-11-21T13:26:07,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742374_1550 (size=12151) 2024-11-21T13:26:07,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742375_1551 (size=12697) 2024-11-21T13:26:07,618 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/f3f524a5c4f24c4d88ae919bad3a908f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/f3f524a5c4f24c4d88ae919bad3a908f 2024-11-21T13:26:07,622 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/C of 8328be8165f9f405996e915498f7765f into f3f524a5c4f24c4d88ae919bad3a908f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:07,622 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:07,622 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/C, priority=13, startTime=1732195567523; duration=0sec 2024-11-21T13:26:07,622 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:07,622 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:C 2024-11-21T13:26:07,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:07,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195627700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:07,704 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:07,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:07,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T13:26:07,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:07,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:07,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:07,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:07,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:07,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T13:26:07,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:07,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195627809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:07,816 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:07,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T13:26:07,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:07,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:07,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:07,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:07,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:08,007 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=228 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/85a093eff63547158a2116d1b659dd04 2024-11-21T13:26:08,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/f7aa9bad3f0346129de8dc5d2c34ad28 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/f7aa9bad3f0346129de8dc5d2c34ad28 2024-11-21T13:26:08,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/f7aa9bad3f0346129de8dc5d2c34ad28, entries=200, sequenceid=228, filesize=14.2 K 2024-11-21T13:26:08,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/918dcf9e0934403cb7dee3459763a3ca as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/918dcf9e0934403cb7dee3459763a3ca 2024-11-21T13:26:08,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/918dcf9e0934403cb7dee3459763a3ca, entries=150, sequenceid=228, filesize=11.9 K 2024-11-21T13:26:08,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/85a093eff63547158a2116d1b659dd04 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/85a093eff63547158a2116d1b659dd04 2024-11-21T13:26:08,023 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:08,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195628019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:08,023 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,024 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:08,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/85a093eff63547158a2116d1b659dd04, entries=150, sequenceid=228, filesize=11.9 K 2024-11-21T13:26:08,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 8328be8165f9f405996e915498f7765f in 502ms, sequenceid=228, compaction requested=false 2024-11-21T13:26:08,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:08,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-21T13:26:08,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:08,036 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T13:26:08,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/767d7499bf0d4e619ed10c604cad8ad3 is 50, key is test_row_0/A:col10/1732195567689/Put/seqid=0 2024-11-21T13:26:08,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742376_1552 (size=12151) 2024-11-21T13:26:08,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T13:26:08,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:08,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:08,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195628340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:08,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195628344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:08,350 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:08,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,351 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:08,443 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/767d7499bf0d4e619ed10c604cad8ad3 2024-11-21T13:26:08,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/9955e18b8b9541d59810883ce4648adc is 50, key is test_row_0/B:col10/1732195567689/Put/seqid=0 2024-11-21T13:26:08,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742377_1553 (size=12151) 2024-11-21T13:26:08,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195628459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,465 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:08,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:08,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195628459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T13:26:08,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195628669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:08,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:08,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195628669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,676 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:08,861 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/9955e18b8b9541d59810883ce4648adc 2024-11-21T13:26:08,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/b35e623138c04a279220e4fa73e14c8d is 50, key is test_row_0/C:col10/1732195567689/Put/seqid=0 2024-11-21T13:26:08,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742378_1554 (size=12151) 2024-11-21T13:26:08,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195628979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:08,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:08,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195628989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:08,993 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:09,287 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/b35e623138c04a279220e4fa73e14c8d 2024-11-21T13:26:09,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/767d7499bf0d4e619ed10c604cad8ad3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/767d7499bf0d4e619ed10c604cad8ad3 2024-11-21T13:26:09,296 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/767d7499bf0d4e619ed10c604cad8ad3, entries=150, sequenceid=256, filesize=11.9 K 2024-11-21T13:26:09,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/9955e18b8b9541d59810883ce4648adc as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/9955e18b8b9541d59810883ce4648adc 2024-11-21T13:26:09,301 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/9955e18b8b9541d59810883ce4648adc, entries=150, sequenceid=256, filesize=11.9 K 2024-11-21T13:26:09,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/b35e623138c04a279220e4fa73e14c8d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/b35e623138c04a279220e4fa73e14c8d 2024-11-21T13:26:09,307 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/b35e623138c04a279220e4fa73e14c8d, entries=150, sequenceid=256, filesize=11.9 K 2024-11-21T13:26:09,307 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 8328be8165f9f405996e915498f7765f in 1271ms, sequenceid=256, compaction requested=true 2024-11-21T13:26:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:09,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-21T13:26:09,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-21T13:26:09,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-21T13:26:09,311 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8860 sec 2024-11-21T13:26:09,313 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.8920 sec 2024-11-21T13:26:09,505 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:09,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-21T13:26:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:09,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/57e0eec9984e432699a1e86f98381089 is 50, key is test_row_0/A:col10/1732195568342/Put/seqid=0 2024-11-21T13:26:09,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742379_1555 (size=12301) 2024-11-21T13:26:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-21T13:26:09,569 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:09,570 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:09,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-21T13:26:09,572 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T13:26:09,573 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:09,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:09,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195629617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:09,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:09,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:09,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:09,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195629622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:09,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:09,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:09,635 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T13:26:09,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T13:26:09,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:09,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:09,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:09,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:09,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:09,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195629729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:09,742 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:09,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195629739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:09,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:09,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:09,749 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:09,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T13:26:09,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:09,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:09,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:09,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:09,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:09,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T13:26:09,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/57e0eec9984e432699a1e86f98381089 2024-11-21T13:26:09,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/db65f2579bbe40e89d683d0dcadba645 is 50, key is test_row_0/B:col10/1732195568342/Put/seqid=0 2024-11-21T13:26:09,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742380_1556 (size=12301) 2024-11-21T13:26:09,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/db65f2579bbe40e89d683d0dcadba645 2024-11-21T13:26:09,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:09,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195629949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:09,954 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:09,955 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:09,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/53b15bd849f34be78f02b40b79e84ad4 is 50, key is test_row_0/C:col10/1732195568342/Put/seqid=0 2024-11-21T13:26:09,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:09,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195629959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:09,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742381_1557 (size=12301) 2024-11-21T13:26:09,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=269 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/53b15bd849f34be78f02b40b79e84ad4 2024-11-21T13:26:09,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/57e0eec9984e432699a1e86f98381089 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/57e0eec9984e432699a1e86f98381089 2024-11-21T13:26:09,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/57e0eec9984e432699a1e86f98381089, entries=150, sequenceid=269, filesize=12.0 K 2024-11-21T13:26:09,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/db65f2579bbe40e89d683d0dcadba645 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/db65f2579bbe40e89d683d0dcadba645 2024-11-21T13:26:09,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/db65f2579bbe40e89d683d0dcadba645, entries=150, sequenceid=269, filesize=12.0 K 2024-11-21T13:26:09,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/53b15bd849f34be78f02b40b79e84ad4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/53b15bd849f34be78f02b40b79e84ad4 2024-11-21T13:26:09,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/53b15bd849f34be78f02b40b79e84ad4, entries=150, sequenceid=269, filesize=12.0 K 2024-11-21T13:26:09,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 8328be8165f9f405996e915498f7765f in 490ms, sequenceid=269, compaction requested=true 2024-11-21T13:26:09,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:09,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:09,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:09,996 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:26:09,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:09,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:09,996 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:26:09,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:09,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:09,999 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51690 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:26:09,999 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:26:09,999 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/A is initiating minor compaction (all files) 2024-11-21T13:26:09,999 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/A in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:09,999 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/2f863e9d891e44468e5e20a11e4c07f2, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/f7aa9bad3f0346129de8dc5d2c34ad28, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/767d7499bf0d4e619ed10c604cad8ad3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/57e0eec9984e432699a1e86f98381089] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=50.5 K 2024-11-21T13:26:10,000 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/B is initiating minor compaction (all files) 2024-11-21T13:26:10,000 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/B in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:10,000 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fddd0311fa784f908150349dc7432d86, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/918dcf9e0934403cb7dee3459763a3ca, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/9955e18b8b9541d59810883ce4648adc, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/db65f2579bbe40e89d683d0dcadba645] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=48.1 K 2024-11-21T13:26:10,000 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting fddd0311fa784f908150349dc7432d86, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732195567433 2024-11-21T13:26:10,001 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f863e9d891e44468e5e20a11e4c07f2, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732195567433 2024-11-21T13:26:10,001 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 918dcf9e0934403cb7dee3459763a3ca, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732195567482 2024-11-21T13:26:10,001 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7aa9bad3f0346129de8dc5d2c34ad28, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732195567482 2024-11-21T13:26:10,001 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9955e18b8b9541d59810883ce4648adc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732195567689 2024-11-21T13:26:10,001 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 767d7499bf0d4e619ed10c604cad8ad3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732195567689 2024-11-21T13:26:10,001 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting db65f2579bbe40e89d683d0dcadba645, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732195568342 2024-11-21T13:26:10,002 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e0eec9984e432699a1e86f98381089, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732195568342 2024-11-21T13:26:10,021 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#B#compaction#467 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:10,021 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#A#compaction#468 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:10,021 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/b1fffe8cb6ed44e28e1d2d2b9596ac1a is 50, key is test_row_0/B:col10/1732195568342/Put/seqid=0 2024-11-21T13:26:10,022 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/65ac3d1832c04c83af97f1211a02a3b3 is 50, key is test_row_0/A:col10/1732195568342/Put/seqid=0 2024-11-21T13:26:10,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-21T13:26:10,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:10,031 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-21T13:26:10,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/e474e3dfe66b49119871c77ae2e7f185 is 50, key is test_row_0/A:col10/1732195569611/Put/seqid=0 2024-11-21T13:26:10,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742383_1559 (size=12983) 2024-11-21T13:26:10,060 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/b1fffe8cb6ed44e28e1d2d2b9596ac1a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/b1fffe8cb6ed44e28e1d2d2b9596ac1a 2024-11-21T13:26:10,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742382_1558 (size=12983) 2024-11-21T13:26:10,069 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8328be8165f9f405996e915498f7765f/B of 8328be8165f9f405996e915498f7765f into b1fffe8cb6ed44e28e1d2d2b9596ac1a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:10,069 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:10,069 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/B, priority=12, startTime=1732195569996; duration=0sec 2024-11-21T13:26:10,069 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:10,069 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:B 2024-11-21T13:26:10,069 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-21T13:26:10,073 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49300 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-21T13:26:10,073 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/C is initiating minor compaction (all files) 2024-11-21T13:26:10,073 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/C in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:10,073 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/f3f524a5c4f24c4d88ae919bad3a908f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/85a093eff63547158a2116d1b659dd04, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/b35e623138c04a279220e4fa73e14c8d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/53b15bd849f34be78f02b40b79e84ad4] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=48.1 K 2024-11-21T13:26:10,073 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f3f524a5c4f24c4d88ae919bad3a908f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=217, earliestPutTs=1732195567433 2024-11-21T13:26:10,074 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 85a093eff63547158a2116d1b659dd04, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=228, earliestPutTs=1732195567482 2024-11-21T13:26:10,074 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b35e623138c04a279220e4fa73e14c8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732195567689 2024-11-21T13:26:10,076 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 53b15bd849f34be78f02b40b79e84ad4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732195568342 2024-11-21T13:26:10,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742384_1560 (size=12301) 2024-11-21T13:26:10,099 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#C#compaction#470 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:10,099 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/00add6d62c3948258a9974037be5a5a6 is 50, key is test_row_0/C:col10/1732195568342/Put/seqid=0 2024-11-21T13:26:10,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742385_1561 (size=12983) 2024-11-21T13:26:10,111 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/00add6d62c3948258a9974037be5a5a6 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/00add6d62c3948258a9974037be5a5a6 2024-11-21T13:26:10,116 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8328be8165f9f405996e915498f7765f/C of 8328be8165f9f405996e915498f7765f into 00add6d62c3948258a9974037be5a5a6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:10,117 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:10,117 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/C, priority=12, startTime=1732195569996; duration=0sec 2024-11-21T13:26:10,117 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:10,117 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:C 2024-11-21T13:26:10,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T13:26:10,267 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:10,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:10,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:10,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195630327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,338 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:10,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:10,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195630329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:10,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:10,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195630449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:10,457 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195630449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:10,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,458 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:10,465 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/65ac3d1832c04c83af97f1211a02a3b3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/65ac3d1832c04c83af97f1211a02a3b3 2024-11-21T13:26:10,469 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 8328be8165f9f405996e915498f7765f/A of 8328be8165f9f405996e915498f7765f into 65ac3d1832c04c83af97f1211a02a3b3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:10,469 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:10,469 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/A, priority=12, startTime=1732195569996; duration=0sec 2024-11-21T13:26:10,470 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:10,470 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:A 2024-11-21T13:26:10,494 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/e474e3dfe66b49119871c77ae2e7f185 2024-11-21T13:26:10,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/4e16c456776742389672c6f950f986a5 is 50, key is test_row_0/B:col10/1732195569611/Put/seqid=0 2024-11-21T13:26:10,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742386_1562 (size=12301) 2024-11-21T13:26:10,566 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/4e16c456776742389672c6f950f986a5 2024-11-21T13:26:10,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/b506ade81f5843b5b2da5b193fdf0424 is 50, key is test_row_0/C:col10/1732195569611/Put/seqid=0 2024-11-21T13:26:10,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742387_1563 (size=12301) 2024-11-21T13:26:10,604 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/b506ade81f5843b5b2da5b193fdf0424 2024-11-21T13:26:10,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/e474e3dfe66b49119871c77ae2e7f185 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/e474e3dfe66b49119871c77ae2e7f185 2024-11-21T13:26:10,624 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/e474e3dfe66b49119871c77ae2e7f185, entries=150, sequenceid=292, filesize=12.0 K 2024-11-21T13:26:10,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/4e16c456776742389672c6f950f986a5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4e16c456776742389672c6f950f986a5 2024-11-21T13:26:10,636 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4e16c456776742389672c6f950f986a5, entries=150, sequenceid=292, filesize=12.0 K 2024-11-21T13:26:10,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/b506ade81f5843b5b2da5b193fdf0424 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/b506ade81f5843b5b2da5b193fdf0424 2024-11-21T13:26:10,651 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/b506ade81f5843b5b2da5b193fdf0424, entries=150, sequenceid=292, filesize=12.0 K 2024-11-21T13:26:10,654 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 8328be8165f9f405996e915498f7765f in 623ms, sequenceid=292, compaction requested=false 2024-11-21T13:26:10,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:10,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:10,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-21T13:26:10,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-21T13:26:10,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-21T13:26:10,658 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0830 sec 2024-11-21T13:26:10,660 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.0880 sec 2024-11-21T13:26:10,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:10,665 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:10,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T13:26:10,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/8c8b9dec8fe740809938f5b9f938feeb is 50, key is test_row_0/A:col10/1732195570325/Put/seqid=0 2024-11-21T13:26:10,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742388_1564 (size=14741) 2024-11-21T13:26:10,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/8c8b9dec8fe740809938f5b9f938feeb 2024-11-21T13:26:10,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-21T13:26:10,711 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:10,712 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:10,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-21T13:26:10,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/951a0b4c92ae42119291a2b6b505b1e7 is 50, key is test_row_0/B:col10/1732195570325/Put/seqid=0 2024-11-21T13:26:10,715 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:10,716 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:10,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:10,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T13:26:10,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742389_1565 (size=12301) 2024-11-21T13:26:10,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/951a0b4c92ae42119291a2b6b505b1e7 2024-11-21T13:26:10,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/dfd0aa7f19a841999e8a2dd4cbc0d090 is 50, key is test_row_0/C:col10/1732195570325/Put/seqid=0 2024-11-21T13:26:10,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742390_1566 (size=12301) 2024-11-21T13:26:10,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:10,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195630762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:10,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:10,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195630768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T13:26:10,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T13:26:10,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:10,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:10,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:10,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:10,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195630879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,888 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:10,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:10,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195630889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:10,896 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:11,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T13:26:11,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:11,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:11,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:11,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:11,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:11,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:11,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T13:26:11,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:11,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195631099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:11,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195631099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,106 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:11,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/dfd0aa7f19a841999e8a2dd4cbc0d090 2024-11-21T13:26:11,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/8c8b9dec8fe740809938f5b9f938feeb as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/8c8b9dec8fe740809938f5b9f938feeb 2024-11-21T13:26:11,152 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/8c8b9dec8fe740809938f5b9f938feeb, entries=200, sequenceid=309, filesize=14.4 K 2024-11-21T13:26:11,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/951a0b4c92ae42119291a2b6b505b1e7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/951a0b4c92ae42119291a2b6b505b1e7 2024-11-21T13:26:11,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/951a0b4c92ae42119291a2b6b505b1e7, entries=150, sequenceid=309, filesize=12.0 K 2024-11-21T13:26:11,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/dfd0aa7f19a841999e8a2dd4cbc0d090 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/dfd0aa7f19a841999e8a2dd4cbc0d090 2024-11-21T13:26:11,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/dfd0aa7f19a841999e8a2dd4cbc0d090, entries=150, sequenceid=309, filesize=12.0 K 2024-11-21T13:26:11,162 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 8328be8165f9f405996e915498f7765f in 496ms, sequenceid=309, compaction requested=true 2024-11-21T13:26:11,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:11,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:11,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:11,162 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:11,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:11,162 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:11,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:11,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:11,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:11,164 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40025 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:11,164 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:11,164 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/A is initiating minor compaction (all files) 2024-11-21T13:26:11,164 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/B is initiating minor compaction (all files) 2024-11-21T13:26:11,164 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/B in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:11,164 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/b1fffe8cb6ed44e28e1d2d2b9596ac1a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4e16c456776742389672c6f950f986a5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/951a0b4c92ae42119291a2b6b505b1e7] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=36.7 K 2024-11-21T13:26:11,164 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/A in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:11,164 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/65ac3d1832c04c83af97f1211a02a3b3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/e474e3dfe66b49119871c77ae2e7f185, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/8c8b9dec8fe740809938f5b9f938feeb] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=39.1 K 2024-11-21T13:26:11,164 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1fffe8cb6ed44e28e1d2d2b9596ac1a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732195568342 2024-11-21T13:26:11,165 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 65ac3d1832c04c83af97f1211a02a3b3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732195568342 2024-11-21T13:26:11,165 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e16c456776742389672c6f950f986a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732195569600 2024-11-21T13:26:11,165 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e474e3dfe66b49119871c77ae2e7f185, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732195569600 2024-11-21T13:26:11,165 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 951a0b4c92ae42119291a2b6b505b1e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732195570311 2024-11-21T13:26:11,165 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c8b9dec8fe740809938f5b9f938feeb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732195570311 2024-11-21T13:26:11,170 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#A#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:11,171 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/897854f2c3a949cb9d17d3e126227a3d is 50, key is test_row_0/A:col10/1732195570325/Put/seqid=0 2024-11-21T13:26:11,173 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#B#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:11,174 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/51a2d10c87a64d2a85fcc3c05c8fc60d is 50, key is test_row_0/B:col10/1732195570325/Put/seqid=0 2024-11-21T13:26:11,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742391_1567 (size=13085) 2024-11-21T13:26:11,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-21T13:26:11,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:11,178 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T13:26:11,184 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/897854f2c3a949cb9d17d3e126227a3d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/897854f2c3a949cb9d17d3e126227a3d 2024-11-21T13:26:11,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/34a0ab0b1c8e4125ad5772ab1d1d47c1 is 50, key is test_row_0/A:col10/1732195570745/Put/seqid=0 2024-11-21T13:26:11,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742392_1568 (size=13085) 2024-11-21T13:26:11,193 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/A of 8328be8165f9f405996e915498f7765f into 897854f2c3a949cb9d17d3e126227a3d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:11,193 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:11,194 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/A, priority=13, startTime=1732195571162; duration=0sec 2024-11-21T13:26:11,194 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:11,194 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:A 2024-11-21T13:26:11,194 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:11,197 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:11,197 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/C is initiating minor compaction (all files) 2024-11-21T13:26:11,197 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/C in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:11,197 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/00add6d62c3948258a9974037be5a5a6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/b506ade81f5843b5b2da5b193fdf0424, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/dfd0aa7f19a841999e8a2dd4cbc0d090] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=36.7 K 2024-11-21T13:26:11,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742393_1569 (size=12301) 2024-11-21T13:26:11,198 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 00add6d62c3948258a9974037be5a5a6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=269, earliestPutTs=1732195568342 2024-11-21T13:26:11,198 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b506ade81f5843b5b2da5b193fdf0424, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1732195569600 2024-11-21T13:26:11,198 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/34a0ab0b1c8e4125ad5772ab1d1d47c1 2024-11-21T13:26:11,198 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting dfd0aa7f19a841999e8a2dd4cbc0d090, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732195570311 2024-11-21T13:26:11,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/dcb835a6ba56487e9ced4d18bdb2d323 is 50, key is test_row_0/B:col10/1732195570745/Put/seqid=0 2024-11-21T13:26:11,211 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#C#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:11,212 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/88129814fd6a4a8abc77ba00b837a5fd is 50, key is test_row_0/C:col10/1732195570325/Put/seqid=0 2024-11-21T13:26:11,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742394_1570 (size=12301) 2024-11-21T13:26:11,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742395_1571 (size=13085) 2024-11-21T13:26:11,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T13:26:11,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:11,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:11,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:11,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195631464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:11,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195631459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:11,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195631579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:11,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195631579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:11,595 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/51a2d10c87a64d2a85fcc3c05c8fc60d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/51a2d10c87a64d2a85fcc3c05c8fc60d 2024-11-21T13:26:11,599 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/B of 8328be8165f9f405996e915498f7765f into 51a2d10c87a64d2a85fcc3c05c8fc60d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:11,599 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:11,599 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/B, priority=13, startTime=1732195571162; duration=0sec 2024-11-21T13:26:11,599 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:11,599 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:B 2024-11-21T13:26:11,621 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/dcb835a6ba56487e9ced4d18bdb2d323 2024-11-21T13:26:11,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/fc14fe7f0c8348bb9ceb9d0fd4a16207 is 50, key is test_row_0/C:col10/1732195570745/Put/seqid=0 2024-11-21T13:26:11,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742396_1572 (size=12301) 2024-11-21T13:26:11,632 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/88129814fd6a4a8abc77ba00b837a5fd as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/88129814fd6a4a8abc77ba00b837a5fd 2024-11-21T13:26:11,638 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/C of 8328be8165f9f405996e915498f7765f into 88129814fd6a4a8abc77ba00b837a5fd(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:11,638 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:11,638 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/C, priority=13, startTime=1732195571162; duration=0sec 2024-11-21T13:26:11,638 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:11,638 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:C 2024-11-21T13:26:11,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:11,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195631789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,796 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:11,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:11,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195631798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:11,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:11,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T13:26:12,029 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/fc14fe7f0c8348bb9ceb9d0fd4a16207 2024-11-21T13:26:12,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/34a0ab0b1c8e4125ad5772ab1d1d47c1 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/34a0ab0b1c8e4125ad5772ab1d1d47c1 2024-11-21T13:26:12,036 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/34a0ab0b1c8e4125ad5772ab1d1d47c1, entries=150, sequenceid=331, filesize=12.0 K 2024-11-21T13:26:12,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/dcb835a6ba56487e9ced4d18bdb2d323 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/dcb835a6ba56487e9ced4d18bdb2d323 2024-11-21T13:26:12,040 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/dcb835a6ba56487e9ced4d18bdb2d323, entries=150, sequenceid=331, filesize=12.0 K 2024-11-21T13:26:12,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/fc14fe7f0c8348bb9ceb9d0fd4a16207 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/fc14fe7f0c8348bb9ceb9d0fd4a16207 2024-11-21T13:26:12,043 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/fc14fe7f0c8348bb9ceb9d0fd4a16207, entries=150, sequenceid=331, filesize=12.0 K 2024-11-21T13:26:12,044 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 8328be8165f9f405996e915498f7765f in 866ms, sequenceid=331, compaction requested=false 2024-11-21T13:26:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:12,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-21T13:26:12,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-21T13:26:12,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-21T13:26:12,047 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3290 sec 2024-11-21T13:26:12,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 1.3350 sec 2024-11-21T13:26:12,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:12,104 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:12,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T13:26:12,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/1463a01478a74ab98cace6c26b42dbe6 is 50, key is test_row_0/A:col10/1732195571462/Put/seqid=0 2024-11-21T13:26:12,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742397_1573 (size=14741) 2024-11-21T13:26:12,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/1463a01478a74ab98cace6c26b42dbe6 2024-11-21T13:26:12,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/45e45c1ca99d430eab0af918ddc4c98a is 50, key is test_row_0/B:col10/1732195571462/Put/seqid=0 2024-11-21T13:26:12,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742398_1574 (size=12301) 2024-11-21T13:26:12,200 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195632192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:12,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195632198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,201 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:12,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195632309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:12,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195632309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,315 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:12,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195632520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:12,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195632520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:12,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,527 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:12,529 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/45e45c1ca99d430eab0af918ddc4c98a 2024-11-21T13:26:12,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/04e6900396ba405bb9382fef0f8db98f is 50, key is test_row_0/C:col10/1732195571462/Put/seqid=0 2024-11-21T13:26:12,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742399_1575 (size=12301) 2024-11-21T13:26:12,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195632829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:12,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195632829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:12,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:12,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-21T13:26:12,860 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:12,860 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-21T13:26:12,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T13:26:12,862 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:12,864 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:12,864 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:12,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T13:26:12,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/04e6900396ba405bb9382fef0f8db98f 2024-11-21T13:26:12,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/1463a01478a74ab98cace6c26b42dbe6 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1463a01478a74ab98cace6c26b42dbe6 2024-11-21T13:26:12,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1463a01478a74ab98cace6c26b42dbe6, entries=200, sequenceid=349, filesize=14.4 K 2024-11-21T13:26:12,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/45e45c1ca99d430eab0af918ddc4c98a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/45e45c1ca99d430eab0af918ddc4c98a 2024-11-21T13:26:12,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/45e45c1ca99d430eab0af918ddc4c98a, entries=150, sequenceid=349, filesize=12.0 K 2024-11-21T13:26:12,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/04e6900396ba405bb9382fef0f8db98f as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/04e6900396ba405bb9382fef0f8db98f 2024-11-21T13:26:12,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/04e6900396ba405bb9382fef0f8db98f, entries=150, sequenceid=349, filesize=12.0 K 2024-11-21T13:26:12,985 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 8328be8165f9f405996e915498f7765f in 881ms, sequenceid=349, compaction requested=true 2024-11-21T13:26:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:12,985 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:12,985 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:12,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:12,986 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:12,986 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/A is initiating minor compaction (all files) 2024-11-21T13:26:12,986 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/A in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:12,986 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/897854f2c3a949cb9d17d3e126227a3d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/34a0ab0b1c8e4125ad5772ab1d1d47c1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1463a01478a74ab98cace6c26b42dbe6] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=39.2 K 2024-11-21T13:26:12,986 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:12,986 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/B is initiating minor compaction (all files) 2024-11-21T13:26:12,986 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/B in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:12,986 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 897854f2c3a949cb9d17d3e126227a3d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732195570311 2024-11-21T13:26:12,986 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/51a2d10c87a64d2a85fcc3c05c8fc60d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/dcb835a6ba56487e9ced4d18bdb2d323, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/45e45c1ca99d430eab0af918ddc4c98a] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=36.8 K 2024-11-21T13:26:12,987 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34a0ab0b1c8e4125ad5772ab1d1d47c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732195570745 2024-11-21T13:26:12,987 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 51a2d10c87a64d2a85fcc3c05c8fc60d, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732195570311 2024-11-21T13:26:12,987 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1463a01478a74ab98cace6c26b42dbe6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732195571441 2024-11-21T13:26:12,987 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting dcb835a6ba56487e9ced4d18bdb2d323, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732195570745 2024-11-21T13:26:12,987 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 45e45c1ca99d430eab0af918ddc4c98a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732195571444 2024-11-21T13:26:12,994 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#B#compaction#485 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:12,994 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#A#compaction#486 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:12,994 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/c6eeb86cd9ff45f69f08fccfe499431b is 50, key is test_row_0/B:col10/1732195571462/Put/seqid=0 2024-11-21T13:26:12,994 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/6e91548ed99543e495d183ad4b4eed52 is 50, key is test_row_0/A:col10/1732195571462/Put/seqid=0 2024-11-21T13:26:12,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742401_1577 (size=13187) 2024-11-21T13:26:12,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742400_1576 (size=13187) 2024-11-21T13:26:13,002 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/6e91548ed99543e495d183ad4b4eed52 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/6e91548ed99543e495d183ad4b4eed52 2024-11-21T13:26:13,003 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/c6eeb86cd9ff45f69f08fccfe499431b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/c6eeb86cd9ff45f69f08fccfe499431b 2024-11-21T13:26:13,007 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/A of 8328be8165f9f405996e915498f7765f into 6e91548ed99543e495d183ad4b4eed52(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:13,007 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:13,007 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/B of 8328be8165f9f405996e915498f7765f into c6eeb86cd9ff45f69f08fccfe499431b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:13,007 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/A, priority=13, startTime=1732195572985; duration=0sec 2024-11-21T13:26:13,007 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:13,007 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/B, priority=13, startTime=1732195572985; duration=0sec 2024-11-21T13:26:13,007 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:13,007 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:13,007 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:A 2024-11-21T13:26:13,007 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:B 2024-11-21T13:26:13,007 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:13,007 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:13,007 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/C is initiating minor compaction (all files) 2024-11-21T13:26:13,008 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/C in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:13,008 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/88129814fd6a4a8abc77ba00b837a5fd, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/fc14fe7f0c8348bb9ceb9d0fd4a16207, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/04e6900396ba405bb9382fef0f8db98f] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=36.8 K 2024-11-21T13:26:13,008 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88129814fd6a4a8abc77ba00b837a5fd, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732195570311 2024-11-21T13:26:13,008 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc14fe7f0c8348bb9ceb9d0fd4a16207, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1732195570745 2024-11-21T13:26:13,008 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04e6900396ba405bb9382fef0f8db98f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732195571444 2024-11-21T13:26:13,014 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#C#compaction#487 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:13,014 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/9c4ba616ede643ae963e54b05aa7cddb is 50, key is test_row_0/C:col10/1732195571462/Put/seqid=0 2024-11-21T13:26:13,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-21T13:26:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:13,016 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T13:26:13,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742402_1578 (size=13187) 2024-11-21T13:26:13,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/34ce7cede3e2447d93c5a6a182ead106 is 50, key is test_row_0/A:col10/1732195572196/Put/seqid=0 2024-11-21T13:26:13,033 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/9c4ba616ede643ae963e54b05aa7cddb as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/9c4ba616ede643ae963e54b05aa7cddb 2024-11-21T13:26:13,041 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/C of 8328be8165f9f405996e915498f7765f into 9c4ba616ede643ae963e54b05aa7cddb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:13,041 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:13,041 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/C, priority=13, startTime=1732195572985; duration=0sec 2024-11-21T13:26:13,041 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:13,041 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:C 2024-11-21T13:26:13,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742403_1579 (size=12301) 2024-11-21T13:26:13,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T13:26:13,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:13,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. as already flushing 2024-11-21T13:26:13,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:13,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195633380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:13,390 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:13,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195633390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:13,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:13,442 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/34ce7cede3e2447d93c5a6a182ead106 2024-11-21T13:26:13,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/4e53fe73071c48f382031cd5b2ebfa28 is 50, key is test_row_0/B:col10/1732195572196/Put/seqid=0 2024-11-21T13:26:13,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742404_1580 (size=12301) 2024-11-21T13:26:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T13:26:13,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195633499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:13,505 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:13,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195633509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:13,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:13,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195633709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:13,715 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:13,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:13,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195633719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:13,722 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:13,854 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/4e53fe73071c48f382031cd5b2ebfa28 2024-11-21T13:26:13,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/35f6724a53d1432285949dea54906722 is 50, key is test_row_0/C:col10/1732195572196/Put/seqid=0 2024-11-21T13:26:13,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742405_1581 (size=12301) 2024-11-21T13:26:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T13:26:14,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:14,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195634019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:14,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:14,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195634029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:14,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,036 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:14,263 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/35f6724a53d1432285949dea54906722 2024-11-21T13:26:14,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/34ce7cede3e2447d93c5a6a182ead106 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/34ce7cede3e2447d93c5a6a182ead106 2024-11-21T13:26:14,272 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/34ce7cede3e2447d93c5a6a182ead106, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T13:26:14,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/4e53fe73071c48f382031cd5b2ebfa28 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4e53fe73071c48f382031cd5b2ebfa28 2024-11-21T13:26:14,275 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4e53fe73071c48f382031cd5b2ebfa28, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T13:26:14,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/35f6724a53d1432285949dea54906722 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/35f6724a53d1432285949dea54906722 2024-11-21T13:26:14,278 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/35f6724a53d1432285949dea54906722, entries=150, sequenceid=372, filesize=12.0 K 2024-11-21T13:26:14,279 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 8328be8165f9f405996e915498f7765f in 1263ms, sequenceid=372, compaction requested=false 2024-11-21T13:26:14,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:14,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:14,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-21T13:26:14,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-21T13:26:14,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-21T13:26:14,281 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-21T13:26:14,283 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.4220 sec 2024-11-21T13:26:14,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:14,533 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:14,533 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T13:26:14,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/1592fa2ecfa24a61a7a186b3a21957d6 is 50, key is test_row_0/A:col10/1732195574532/Put/seqid=0 2024-11-21T13:26:14,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742406_1582 (size=14741) 2024-11-21T13:26:14,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/1592fa2ecfa24a61a7a186b3a21957d6 2024-11-21T13:26:14,546 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/950bf79923514431b3780c11ece732ff is 50, key is test_row_0/B:col10/1732195574532/Put/seqid=0 2024-11-21T13:26:14,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742407_1583 (size=12301) 2024-11-21T13:26:14,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:14,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195634605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:14,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:14,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195634606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:14,658 INFO [Thread-2215 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2215. 2024-11-21T13:26:14,658 DEBUG [Thread-2215 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:14,658 DEBUG [Thread-2215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:14,658 INFO [Thread-2217 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2217. 2024-11-21T13:26:14,658 DEBUG [Thread-2217 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:14,658 DEBUG [Thread-2217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:14,659 DEBUG [Thread-2215 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:14,659 DEBUG [Thread-2217 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:14,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:14,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:14,659 INFO [Thread-2219 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2219. 2024-11-21T13:26:14,659 DEBUG [Thread-2219 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:14,659 DEBUG [Thread-2219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:14,659 DEBUG [Thread-2219 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:14,659 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:14,666 INFO [Thread-2223 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2223. 2024-11-21T13:26:14,666 DEBUG [Thread-2223 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:14,666 DEBUG [Thread-2223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:14,666 DEBUG [Thread-2223 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:14,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:14,669 INFO [Thread-2221 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2221. 2024-11-21T13:26:14,669 DEBUG [Thread-2221 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:14,669 DEBUG [Thread-2221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:14,669 DEBUG [Thread-2221 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:14,669 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:14,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:14,719 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195634719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:14,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195634719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:14,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:14,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195634929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195634929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:14,950 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/950bf79923514431b3780c11ece732ff 2024-11-21T13:26:14,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/8ba7f048ae2941928c9196cb97ebee06 is 50, key is test_row_0/C:col10/1732195574532/Put/seqid=0 2024-11-21T13:26:14,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742408_1584 (size=12301) 2024-11-21T13:26:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-21T13:26:15,009 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:15,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:15,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49032 deadline: 1732195635239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49048 deadline: 1732195635239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:15,239 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:15,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:15,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48986 deadline: 1732195635319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:15,320 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18287 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:15,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:15,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:15,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:15,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49000 deadline: 1732195635349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:15,350 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18324 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:15,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:15,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:15,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:15,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/8ba7f048ae2941928c9196cb97ebee06 2024-11-21T13:26:15,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/1592fa2ecfa24a61a7a186b3a21957d6 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1592fa2ecfa24a61a7a186b3a21957d6 2024-11-21T13:26:15,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1592fa2ecfa24a61a7a186b3a21957d6, entries=200, sequenceid=389, filesize=14.4 K 2024-11-21T13:26:15,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/950bf79923514431b3780c11ece732ff as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/950bf79923514431b3780c11ece732ff 2024-11-21T13:26:15,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/950bf79923514431b3780c11ece732ff, entries=150, sequenceid=389, filesize=12.0 K 2024-11-21T13:26:15,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/8ba7f048ae2941928c9196cb97ebee06 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/8ba7f048ae2941928c9196cb97ebee06 2024-11-21T13:26:15,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48972 deadline: 1732195635369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:15,369 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 8328be8165f9f405996e915498f7765f of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18339 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 , the old value is region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:15,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., hostname=4fe6efa1e611,33573,1732195430851, seqNum=2 because the exception is null or not the one we care about 2024-11-21T13:26:15,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/8ba7f048ae2941928c9196cb97ebee06, entries=150, sequenceid=389, filesize=12.0 K 2024-11-21T13:26:15,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 8328be8165f9f405996e915498f7765f in 839ms, sequenceid=389, compaction requested=true 2024-11-21T13:26:15,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:15,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:15,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:15,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:15,372 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:15,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:15,372 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:15,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 8328be8165f9f405996e915498f7765f:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:15,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:15,373 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:15,373 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:15,373 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/B is initiating minor compaction (all files) 2024-11-21T13:26:15,373 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/A is initiating minor compaction (all files) 2024-11-21T13:26:15,373 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/B in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:15,373 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/A in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:15,373 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/c6eeb86cd9ff45f69f08fccfe499431b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4e53fe73071c48f382031cd5b2ebfa28, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/950bf79923514431b3780c11ece732ff] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=36.9 K 2024-11-21T13:26:15,373 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/6e91548ed99543e495d183ad4b4eed52, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/34ce7cede3e2447d93c5a6a182ead106, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1592fa2ecfa24a61a7a186b3a21957d6] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=39.3 K 2024-11-21T13:26:15,374 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting c6eeb86cd9ff45f69f08fccfe499431b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732195571444 2024-11-21T13:26:15,374 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e91548ed99543e495d183ad4b4eed52, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732195571444 2024-11-21T13:26:15,374 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34ce7cede3e2447d93c5a6a182ead106, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732195572155 2024-11-21T13:26:15,374 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e53fe73071c48f382031cd5b2ebfa28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732195572155 2024-11-21T13:26:15,374 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1592fa2ecfa24a61a7a186b3a21957d6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732195573355 2024-11-21T13:26:15,374 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 950bf79923514431b3780c11ece732ff, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732195573378 2024-11-21T13:26:15,379 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#A#compaction#494 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:15,379 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/a842b98868b64c8f95f1bf3971ccfd9c is 50, key is test_row_0/A:col10/1732195574532/Put/seqid=0 2024-11-21T13:26:15,380 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#B#compaction#495 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:15,380 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/fe6c77cd19b94e019ea2210a67a48a74 is 50, key is test_row_0/B:col10/1732195574532/Put/seqid=0 2024-11-21T13:26:15,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742409_1585 (size=13289) 2024-11-21T13:26:15,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742410_1586 (size=13289) 2024-11-21T13:26:15,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:15,751 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:15,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T13:26:15,751 INFO [Thread-2208 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2208. 2024-11-21T13:26:15,751 INFO [Thread-2204 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2204. 2024-11-21T13:26:15,751 DEBUG [Thread-2208 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:15,751 DEBUG [Thread-2208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:15,751 DEBUG [Thread-2204 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:15,751 DEBUG [Thread-2204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:15,751 DEBUG [Thread-2208 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:15,751 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:15,751 DEBUG [Thread-2204 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:15,751 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:15,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/384f3d0fefde4e2184783fee19f4d334 is 50, key is test_row_0/A:col10/1732195575750/Put/seqid=0 2024-11-21T13:26:15,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742411_1587 (size=12301) 2024-11-21T13:26:15,787 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/fe6c77cd19b94e019ea2210a67a48a74 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fe6c77cd19b94e019ea2210a67a48a74 2024-11-21T13:26:15,787 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/a842b98868b64c8f95f1bf3971ccfd9c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/a842b98868b64c8f95f1bf3971ccfd9c 2024-11-21T13:26:15,791 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/B of 8328be8165f9f405996e915498f7765f into fe6c77cd19b94e019ea2210a67a48a74(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:15,791 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/A of 8328be8165f9f405996e915498f7765f into a842b98868b64c8f95f1bf3971ccfd9c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:15,791 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:15,791 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:15,791 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/B, priority=13, startTime=1732195575372; duration=0sec 2024-11-21T13:26:15,791 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/A, priority=13, startTime=1732195575372; duration=0sec 2024-11-21T13:26:15,791 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:15,791 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:B 2024-11-21T13:26:15,791 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:15,791 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:A 2024-11-21T13:26:15,791 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:15,791 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:15,791 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 8328be8165f9f405996e915498f7765f/C is initiating minor compaction (all files) 2024-11-21T13:26:15,791 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 8328be8165f9f405996e915498f7765f/C in TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:15,791 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/9c4ba616ede643ae963e54b05aa7cddb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/35f6724a53d1432285949dea54906722, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/8ba7f048ae2941928c9196cb97ebee06] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp, totalSize=36.9 K 2024-11-21T13:26:15,792 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c4ba616ede643ae963e54b05aa7cddb, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732195571444 2024-11-21T13:26:15,792 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 35f6724a53d1432285949dea54906722, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1732195572155 2024-11-21T13:26:15,792 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ba7f048ae2941928c9196cb97ebee06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1732195573378 2024-11-21T13:26:15,797 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 8328be8165f9f405996e915498f7765f#C#compaction#497 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:15,797 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/bcb054efac57484aa1bc1acc0f1e0e30 is 50, key is test_row_0/C:col10/1732195574532/Put/seqid=0 2024-11-21T13:26:15,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742412_1588 (size=13289) 2024-11-21T13:26:16,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/384f3d0fefde4e2184783fee19f4d334 2024-11-21T13:26:16,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/1a318d57cd6440e5a90bc48a95e27c37 is 50, key is test_row_0/B:col10/1732195575750/Put/seqid=0 2024-11-21T13:26:16,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742413_1589 (size=12301) 2024-11-21T13:26:16,203 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/bcb054efac57484aa1bc1acc0f1e0e30 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/bcb054efac57484aa1bc1acc0f1e0e30 2024-11-21T13:26:16,208 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 8328be8165f9f405996e915498f7765f/C of 8328be8165f9f405996e915498f7765f into bcb054efac57484aa1bc1acc0f1e0e30(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:16,208 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:16,208 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., storeName=8328be8165f9f405996e915498f7765f/C, priority=13, startTime=1732195575372; duration=0sec 2024-11-21T13:26:16,208 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:16,208 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 8328be8165f9f405996e915498f7765f:C 2024-11-21T13:26:16,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/1a318d57cd6440e5a90bc48a95e27c37 2024-11-21T13:26:16,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/3caddc0098eb4f7595217b7fdb6f7701 is 50, key is test_row_0/C:col10/1732195575750/Put/seqid=0 2024-11-21T13:26:16,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742414_1590 (size=12301) 2024-11-21T13:26:16,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/3caddc0098eb4f7595217b7fdb6f7701 2024-11-21T13:26:16,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/384f3d0fefde4e2184783fee19f4d334 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/384f3d0fefde4e2184783fee19f4d334 2024-11-21T13:26:16,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/384f3d0fefde4e2184783fee19f4d334, entries=150, sequenceid=411, filesize=12.0 K 2024-11-21T13:26:16,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/1a318d57cd6440e5a90bc48a95e27c37 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/1a318d57cd6440e5a90bc48a95e27c37 2024-11-21T13:26:16,990 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/1a318d57cd6440e5a90bc48a95e27c37, entries=150, sequenceid=411, filesize=12.0 K 2024-11-21T13:26:16,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/3caddc0098eb4f7595217b7fdb6f7701 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/3caddc0098eb4f7595217b7fdb6f7701 2024-11-21T13:26:16,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/3caddc0098eb4f7595217b7fdb6f7701, entries=150, sequenceid=411, filesize=12.0 K 2024-11-21T13:26:16,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=0 B/0 for 8328be8165f9f405996e915498f7765f in 1244ms, sequenceid=411, compaction requested=false 2024-11-21T13:26:16,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 8328be8165f9f405996e915498f7765f: 2024-11-21T13:26:19,230 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T13:26:25,360 INFO [Thread-2210 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2210. 2024-11-21T13:26:25,360 DEBUG [Thread-2210 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:25,360 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:25,360 DEBUG [Thread-2210 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:25,360 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:25,410 INFO [Thread-2206 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2206. 2024-11-21T13:26:25,410 DEBUG [Thread-2206 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:25,410 DEBUG [Thread-2206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:25,410 DEBUG [Thread-2206 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:25,410 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:25,420 INFO [Thread-2212 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2212. 2024-11-21T13:26:25,420 DEBUG [Thread-2212 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:25,420 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:25,420 DEBUG [Thread-2212 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:25,420 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:25,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T13:26:25,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-11-21T13:26:25,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 15 2024-11-21T13:26:25,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 158 2024-11-21T13:26:25,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 14 2024-11-21T13:26:25,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-21T13:26:25,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T13:26:25,420 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T13:26:25,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1701 2024-11-21T13:26:25,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5103 rows 2024-11-21T13:26:25,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1700 2024-11-21T13:26:25,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5096 rows 2024-11-21T13:26:25,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1693 2024-11-21T13:26:25,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5078 rows 2024-11-21T13:26:25,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1699 2024-11-21T13:26:25,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5094 rows 2024-11-21T13:26:25,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1694 2024-11-21T13:26:25,421 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 5081 rows 2024-11-21T13:26:25,421 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T13:26:25,421 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T13:26:25,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:25,421 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:25,421 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:25,421 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T13:26:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T13:26:25,424 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195585424"}]},"ts":"1732195585424"} 2024-11-21T13:26:25,425 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T13:26:25,425 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T13:26:25,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T13:26:25,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8328be8165f9f405996e915498f7765f, UNASSIGN}] 2024-11-21T13:26:25,427 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8328be8165f9f405996e915498f7765f, UNASSIGN 2024-11-21T13:26:25,427 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=8328be8165f9f405996e915498f7765f, regionState=CLOSING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:25,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=8328be8165f9f405996e915498f7765f, UNASSIGN because future has completed 2024-11-21T13:26:25,429 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T13:26:25,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:26:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T13:26:25,580 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(122): Close 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:25,580 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T13:26:25,581 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1722): Closing 8328be8165f9f405996e915498f7765f, disabling compactions & flushes 2024-11-21T13:26:25,581 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:25,581 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:25,581 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. after waiting 0 ms 2024-11-21T13:26:25,581 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:25,581 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(2902): Flushing 8328be8165f9f405996e915498f7765f 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-21T13:26:25,584 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/386436acbd484303965058b9fbfbf3e4 is 50, key is test_row_0/A:col10/1732195585409/Put/seqid=0 2024-11-21T13:26:25,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742415_1591 (size=9857) 2024-11-21T13:26:25,587 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/386436acbd484303965058b9fbfbf3e4 2024-11-21T13:26:25,593 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/c4ceef2f28854f488f0dc844fe316375 is 50, key is test_row_0/B:col10/1732195585409/Put/seqid=0 2024-11-21T13:26:25,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742416_1592 (size=9857) 2024-11-21T13:26:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T13:26:25,996 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/c4ceef2f28854f488f0dc844fe316375 2024-11-21T13:26:26,001 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/3bfcdada3c834b79a8d1e4e434058deb is 50, key is test_row_0/C:col10/1732195585409/Put/seqid=0 2024-11-21T13:26:26,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742417_1593 (size=9857) 2024-11-21T13:26:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T13:26:26,406 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/3bfcdada3c834b79a8d1e4e434058deb 2024-11-21T13:26:26,410 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/A/386436acbd484303965058b9fbfbf3e4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/386436acbd484303965058b9fbfbf3e4 2024-11-21T13:26:26,412 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/386436acbd484303965058b9fbfbf3e4, entries=100, sequenceid=420, filesize=9.6 K 2024-11-21T13:26:26,413 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/B/c4ceef2f28854f488f0dc844fe316375 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/c4ceef2f28854f488f0dc844fe316375 2024-11-21T13:26:26,415 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/c4ceef2f28854f488f0dc844fe316375, entries=100, sequenceid=420, filesize=9.6 K 2024-11-21T13:26:26,416 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/.tmp/C/3bfcdada3c834b79a8d1e4e434058deb as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/3bfcdada3c834b79a8d1e4e434058deb 2024-11-21T13:26:26,418 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/3bfcdada3c834b79a8d1e4e434058deb, entries=100, sequenceid=420, filesize=9.6 K 2024-11-21T13:26:26,418 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 8328be8165f9f405996e915498f7765f in 837ms, sequenceid=420, compaction requested=true 2024-11-21T13:26:26,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/65c22ea15c774a6b88d3f3f516decc98, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/99b02f101b0f409c997f5130f0000a4b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/7258fb3ea5174c3dac08047fd2319fce, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1a40bebbd37b4a3a938231a0aa324808, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/6b6badf173eb48ddab1db18fb3679e79, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/a4193ebd852041478be08c9938b0f20a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/0e7b2526d76c4556a9eee03ce8c49ae9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/66b42b10c79e460a93d4f7cf77d5c7be, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/e63447d0fa8147e58a1b6c603b02e382, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/89d088e0085b44e890b07ce40c8f3360, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/c6175a9b081d47789979894ebbf0d362, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/0b037d9de7584e818dc645717e4e0231, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/d5494c0bec4446a490b1ae360271370b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1cf40fb34cf34e2a9de8b96aecae96d7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/76620f5ea584460f98ff894717fa877f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/2f863e9d891e44468e5e20a11e4c07f2, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/d6c42c1b22f94bdb927cf1f8fdce1a03, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/f7aa9bad3f0346129de8dc5d2c34ad28, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/767d7499bf0d4e619ed10c604cad8ad3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/65ac3d1832c04c83af97f1211a02a3b3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/57e0eec9984e432699a1e86f98381089, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/e474e3dfe66b49119871c77ae2e7f185, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/8c8b9dec8fe740809938f5b9f938feeb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/897854f2c3a949cb9d17d3e126227a3d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/34a0ab0b1c8e4125ad5772ab1d1d47c1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1463a01478a74ab98cace6c26b42dbe6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/6e91548ed99543e495d183ad4b4eed52, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/34ce7cede3e2447d93c5a6a182ead106, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1592fa2ecfa24a61a7a186b3a21957d6] to archive 2024-11-21T13:26:26,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:26:26,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/65c22ea15c774a6b88d3f3f516decc98 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/65c22ea15c774a6b88d3f3f516decc98 2024-11-21T13:26:26,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/99b02f101b0f409c997f5130f0000a4b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/99b02f101b0f409c997f5130f0000a4b 2024-11-21T13:26:26,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/7258fb3ea5174c3dac08047fd2319fce to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/7258fb3ea5174c3dac08047fd2319fce 2024-11-21T13:26:26,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1a40bebbd37b4a3a938231a0aa324808 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1a40bebbd37b4a3a938231a0aa324808 2024-11-21T13:26:26,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/6b6badf173eb48ddab1db18fb3679e79 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/6b6badf173eb48ddab1db18fb3679e79 2024-11-21T13:26:26,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/a4193ebd852041478be08c9938b0f20a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/a4193ebd852041478be08c9938b0f20a 2024-11-21T13:26:26,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/0e7b2526d76c4556a9eee03ce8c49ae9 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/0e7b2526d76c4556a9eee03ce8c49ae9 2024-11-21T13:26:26,426 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/66b42b10c79e460a93d4f7cf77d5c7be to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/66b42b10c79e460a93d4f7cf77d5c7be 2024-11-21T13:26:26,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/e63447d0fa8147e58a1b6c603b02e382 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/e63447d0fa8147e58a1b6c603b02e382 2024-11-21T13:26:26,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/89d088e0085b44e890b07ce40c8f3360 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/89d088e0085b44e890b07ce40c8f3360 2024-11-21T13:26:26,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/c6175a9b081d47789979894ebbf0d362 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/c6175a9b081d47789979894ebbf0d362 2024-11-21T13:26:26,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/0b037d9de7584e818dc645717e4e0231 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/0b037d9de7584e818dc645717e4e0231 2024-11-21T13:26:26,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/d5494c0bec4446a490b1ae360271370b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/d5494c0bec4446a490b1ae360271370b 2024-11-21T13:26:26,430 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1cf40fb34cf34e2a9de8b96aecae96d7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1cf40fb34cf34e2a9de8b96aecae96d7 2024-11-21T13:26:26,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/76620f5ea584460f98ff894717fa877f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/76620f5ea584460f98ff894717fa877f 2024-11-21T13:26:26,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/2f863e9d891e44468e5e20a11e4c07f2 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/2f863e9d891e44468e5e20a11e4c07f2 2024-11-21T13:26:26,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/d6c42c1b22f94bdb927cf1f8fdce1a03 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/d6c42c1b22f94bdb927cf1f8fdce1a03 2024-11-21T13:26:26,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/f7aa9bad3f0346129de8dc5d2c34ad28 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/f7aa9bad3f0346129de8dc5d2c34ad28 2024-11-21T13:26:26,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/767d7499bf0d4e619ed10c604cad8ad3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/767d7499bf0d4e619ed10c604cad8ad3 2024-11-21T13:26:26,434 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/65ac3d1832c04c83af97f1211a02a3b3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/65ac3d1832c04c83af97f1211a02a3b3 2024-11-21T13:26:26,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/57e0eec9984e432699a1e86f98381089 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/57e0eec9984e432699a1e86f98381089 2024-11-21T13:26:26,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/e474e3dfe66b49119871c77ae2e7f185 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/e474e3dfe66b49119871c77ae2e7f185 2024-11-21T13:26:26,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/8c8b9dec8fe740809938f5b9f938feeb to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/8c8b9dec8fe740809938f5b9f938feeb 2024-11-21T13:26:26,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/897854f2c3a949cb9d17d3e126227a3d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/897854f2c3a949cb9d17d3e126227a3d 2024-11-21T13:26:26,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/34a0ab0b1c8e4125ad5772ab1d1d47c1 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/34a0ab0b1c8e4125ad5772ab1d1d47c1 2024-11-21T13:26:26,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1463a01478a74ab98cace6c26b42dbe6 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1463a01478a74ab98cace6c26b42dbe6 2024-11-21T13:26:26,438 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/6e91548ed99543e495d183ad4b4eed52 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/6e91548ed99543e495d183ad4b4eed52 2024-11-21T13:26:26,441 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/34ce7cede3e2447d93c5a6a182ead106 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/34ce7cede3e2447d93c5a6a182ead106 2024-11-21T13:26:26,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1592fa2ecfa24a61a7a186b3a21957d6 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/1592fa2ecfa24a61a7a186b3a21957d6 2024-11-21T13:26:26,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/8567e380ba4e41a189ce389b47cfca36, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/357cf3454de248dda61c7cbd9bed9a10, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/d67a4ad246b4452fb6616352aaee58b5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/e01f13dc8f5442b4bf57b43849a33304, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/a04c8810472c4acdbb9489fee1c169a6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4f38522e0be84e8dbc21b11dd96535e3, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/8c8bd421be794a36a3289cfce011d1e7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/d79a89bfe63546e0b156f8d4f1daa2b0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fa7f2cdcf50a464cb7909cd84719e45c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fd185279849841948cdf326f14048485, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/95cdf4e1032443129384d51a0d7dfb3c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/893f9c8219c94ec79953a23a1f5dd842, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/443c8d3f72d343b98106ec6ff6e41822, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4a744a950fda4b7b9e0984dff71ca29d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/af7c46cb9f1b4d16a63ca539c8ba5af8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fddd0311fa784f908150349dc7432d86, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/1a33aa2ae11b4987a4e5cc57f068f77f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/918dcf9e0934403cb7dee3459763a3ca, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/9955e18b8b9541d59810883ce4648adc, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/b1fffe8cb6ed44e28e1d2d2b9596ac1a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/db65f2579bbe40e89d683d0dcadba645, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4e16c456776742389672c6f950f986a5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/51a2d10c87a64d2a85fcc3c05c8fc60d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/951a0b4c92ae42119291a2b6b505b1e7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/dcb835a6ba56487e9ced4d18bdb2d323, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/c6eeb86cd9ff45f69f08fccfe499431b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/45e45c1ca99d430eab0af918ddc4c98a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4e53fe73071c48f382031cd5b2ebfa28, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/950bf79923514431b3780c11ece732ff] to archive 2024-11-21T13:26:26,445 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:26:26,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/8567e380ba4e41a189ce389b47cfca36 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/8567e380ba4e41a189ce389b47cfca36 2024-11-21T13:26:26,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/357cf3454de248dda61c7cbd9bed9a10 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/357cf3454de248dda61c7cbd9bed9a10 2024-11-21T13:26:26,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/d67a4ad246b4452fb6616352aaee58b5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/d67a4ad246b4452fb6616352aaee58b5 2024-11-21T13:26:26,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/e01f13dc8f5442b4bf57b43849a33304 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/e01f13dc8f5442b4bf57b43849a33304 2024-11-21T13:26:26,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/a04c8810472c4acdbb9489fee1c169a6 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/a04c8810472c4acdbb9489fee1c169a6 2024-11-21T13:26:26,449 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4f38522e0be84e8dbc21b11dd96535e3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4f38522e0be84e8dbc21b11dd96535e3 2024-11-21T13:26:26,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/8c8bd421be794a36a3289cfce011d1e7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/8c8bd421be794a36a3289cfce011d1e7 2024-11-21T13:26:26,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/d79a89bfe63546e0b156f8d4f1daa2b0 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/d79a89bfe63546e0b156f8d4f1daa2b0 2024-11-21T13:26:26,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fa7f2cdcf50a464cb7909cd84719e45c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fa7f2cdcf50a464cb7909cd84719e45c 2024-11-21T13:26:26,452 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fd185279849841948cdf326f14048485 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fd185279849841948cdf326f14048485 2024-11-21T13:26:26,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/95cdf4e1032443129384d51a0d7dfb3c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/95cdf4e1032443129384d51a0d7dfb3c 2024-11-21T13:26:26,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/893f9c8219c94ec79953a23a1f5dd842 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/893f9c8219c94ec79953a23a1f5dd842 2024-11-21T13:26:26,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/443c8d3f72d343b98106ec6ff6e41822 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/443c8d3f72d343b98106ec6ff6e41822 2024-11-21T13:26:26,455 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4a744a950fda4b7b9e0984dff71ca29d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4a744a950fda4b7b9e0984dff71ca29d 2024-11-21T13:26:26,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/af7c46cb9f1b4d16a63ca539c8ba5af8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/af7c46cb9f1b4d16a63ca539c8ba5af8 2024-11-21T13:26:26,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fddd0311fa784f908150349dc7432d86 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fddd0311fa784f908150349dc7432d86 2024-11-21T13:26:26,457 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/1a33aa2ae11b4987a4e5cc57f068f77f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/1a33aa2ae11b4987a4e5cc57f068f77f 2024-11-21T13:26:26,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/918dcf9e0934403cb7dee3459763a3ca to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/918dcf9e0934403cb7dee3459763a3ca 2024-11-21T13:26:26,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/9955e18b8b9541d59810883ce4648adc to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/9955e18b8b9541d59810883ce4648adc 2024-11-21T13:26:26,460 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/b1fffe8cb6ed44e28e1d2d2b9596ac1a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/b1fffe8cb6ed44e28e1d2d2b9596ac1a 2024-11-21T13:26:26,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/db65f2579bbe40e89d683d0dcadba645 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/db65f2579bbe40e89d683d0dcadba645 2024-11-21T13:26:26,462 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4e16c456776742389672c6f950f986a5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4e16c456776742389672c6f950f986a5 2024-11-21T13:26:26,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/51a2d10c87a64d2a85fcc3c05c8fc60d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/51a2d10c87a64d2a85fcc3c05c8fc60d 2024-11-21T13:26:26,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/951a0b4c92ae42119291a2b6b505b1e7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/951a0b4c92ae42119291a2b6b505b1e7 2024-11-21T13:26:26,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/dcb835a6ba56487e9ced4d18bdb2d323 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/dcb835a6ba56487e9ced4d18bdb2d323 2024-11-21T13:26:26,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/c6eeb86cd9ff45f69f08fccfe499431b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/c6eeb86cd9ff45f69f08fccfe499431b 2024-11-21T13:26:26,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/45e45c1ca99d430eab0af918ddc4c98a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/45e45c1ca99d430eab0af918ddc4c98a 2024-11-21T13:26:26,466 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4e53fe73071c48f382031cd5b2ebfa28 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/4e53fe73071c48f382031cd5b2ebfa28 2024-11-21T13:26:26,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/950bf79923514431b3780c11ece732ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/950bf79923514431b3780c11ece732ff 2024-11-21T13:26:26,468 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/5383bce47caf41b2ba0d55b9169c9998, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/daa7bb9386dd45ed833f3747161a814b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2c47c3cdd68e4c1b9208dcefb1ebeb8b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2cd09f6e2f9b4594b7b19b9e0627053f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/f807f763b65b41ba8a7c10a265553dca, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/4cb9a36aa1e54d2f823b5b5683559691, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2539795f06a84bc3a8723ea81f4e502d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/421d3b6c46734bd18b7fd363942d2f9f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/848dee9956934d4396d0f465f44bbba5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/10242f7abe1f48749c6662318db5a13c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/127edb3550a842c7b83e6c26718205a6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/79522ebb42034b828e51c746e9ce3510, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/5d970b6a099742d491d58a770659cdda, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/ee5893e40c264b79b3a95d02fda31e4d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/da5b70827dd34d08ac9975019934d214, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/f3f524a5c4f24c4d88ae919bad3a908f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/59afa3323c464903a8bfd00f62576b05, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/85a093eff63547158a2116d1b659dd04, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/b35e623138c04a279220e4fa73e14c8d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/00add6d62c3948258a9974037be5a5a6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/53b15bd849f34be78f02b40b79e84ad4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/b506ade81f5843b5b2da5b193fdf0424, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/88129814fd6a4a8abc77ba00b837a5fd, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/dfd0aa7f19a841999e8a2dd4cbc0d090, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/fc14fe7f0c8348bb9ceb9d0fd4a16207, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/9c4ba616ede643ae963e54b05aa7cddb, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/04e6900396ba405bb9382fef0f8db98f, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/35f6724a53d1432285949dea54906722, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/8ba7f048ae2941928c9196cb97ebee06] to archive 2024-11-21T13:26:26,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:26:26,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/5383bce47caf41b2ba0d55b9169c9998 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/5383bce47caf41b2ba0d55b9169c9998 2024-11-21T13:26:26,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/daa7bb9386dd45ed833f3747161a814b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/daa7bb9386dd45ed833f3747161a814b 2024-11-21T13:26:26,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2c47c3cdd68e4c1b9208dcefb1ebeb8b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2c47c3cdd68e4c1b9208dcefb1ebeb8b 2024-11-21T13:26:26,472 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2cd09f6e2f9b4594b7b19b9e0627053f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2cd09f6e2f9b4594b7b19b9e0627053f 2024-11-21T13:26:26,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/f807f763b65b41ba8a7c10a265553dca to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/f807f763b65b41ba8a7c10a265553dca 2024-11-21T13:26:26,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/4cb9a36aa1e54d2f823b5b5683559691 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/4cb9a36aa1e54d2f823b5b5683559691 2024-11-21T13:26:26,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2539795f06a84bc3a8723ea81f4e502d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/2539795f06a84bc3a8723ea81f4e502d 2024-11-21T13:26:26,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/421d3b6c46734bd18b7fd363942d2f9f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/421d3b6c46734bd18b7fd363942d2f9f 2024-11-21T13:26:26,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/848dee9956934d4396d0f465f44bbba5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/848dee9956934d4396d0f465f44bbba5 2024-11-21T13:26:26,476 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/10242f7abe1f48749c6662318db5a13c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/10242f7abe1f48749c6662318db5a13c 2024-11-21T13:26:26,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/127edb3550a842c7b83e6c26718205a6 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/127edb3550a842c7b83e6c26718205a6 2024-11-21T13:26:26,477 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/79522ebb42034b828e51c746e9ce3510 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/79522ebb42034b828e51c746e9ce3510 2024-11-21T13:26:26,478 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/5d970b6a099742d491d58a770659cdda to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/5d970b6a099742d491d58a770659cdda 2024-11-21T13:26:26,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/ee5893e40c264b79b3a95d02fda31e4d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/ee5893e40c264b79b3a95d02fda31e4d 2024-11-21T13:26:26,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/da5b70827dd34d08ac9975019934d214 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/da5b70827dd34d08ac9975019934d214 2024-11-21T13:26:26,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/f3f524a5c4f24c4d88ae919bad3a908f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/f3f524a5c4f24c4d88ae919bad3a908f 2024-11-21T13:26:26,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/59afa3323c464903a8bfd00f62576b05 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/59afa3323c464903a8bfd00f62576b05 2024-11-21T13:26:26,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/85a093eff63547158a2116d1b659dd04 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/85a093eff63547158a2116d1b659dd04 2024-11-21T13:26:26,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/b35e623138c04a279220e4fa73e14c8d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/b35e623138c04a279220e4fa73e14c8d 2024-11-21T13:26:26,483 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/00add6d62c3948258a9974037be5a5a6 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/00add6d62c3948258a9974037be5a5a6 2024-11-21T13:26:26,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/53b15bd849f34be78f02b40b79e84ad4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/53b15bd849f34be78f02b40b79e84ad4 2024-11-21T13:26:26,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/b506ade81f5843b5b2da5b193fdf0424 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/b506ade81f5843b5b2da5b193fdf0424 2024-11-21T13:26:26,486 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/88129814fd6a4a8abc77ba00b837a5fd to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/88129814fd6a4a8abc77ba00b837a5fd 2024-11-21T13:26:26,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/dfd0aa7f19a841999e8a2dd4cbc0d090 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/dfd0aa7f19a841999e8a2dd4cbc0d090 2024-11-21T13:26:26,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/fc14fe7f0c8348bb9ceb9d0fd4a16207 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/fc14fe7f0c8348bb9ceb9d0fd4a16207 2024-11-21T13:26:26,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/9c4ba616ede643ae963e54b05aa7cddb to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/9c4ba616ede643ae963e54b05aa7cddb 2024-11-21T13:26:26,489 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/04e6900396ba405bb9382fef0f8db98f to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/04e6900396ba405bb9382fef0f8db98f 2024-11-21T13:26:26,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/35f6724a53d1432285949dea54906722 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/35f6724a53d1432285949dea54906722 2024-11-21T13:26:26,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/8ba7f048ae2941928c9196cb97ebee06 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/8ba7f048ae2941928c9196cb97ebee06 2024-11-21T13:26:26,494 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/recovered.edits/423.seqid, newMaxSeqId=423, maxSeqId=1 2024-11-21T13:26:26,495 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. 2024-11-21T13:26:26,495 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] regionserver.HRegion(1676): Region close journal for 8328be8165f9f405996e915498f7765f: Waiting for close lock at 1732195585581Running coprocessor pre-close hooks at 1732195585581Disabling compacts and flushes for region at 1732195585581Disabling writes for close at 1732195585581Obtaining lock to block concurrent updates at 1732195585581Preparing flush snapshotting stores in 8328be8165f9f405996e915498f7765f at 1732195585581Finished memstore snapshotting TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1732195585581Flushing stores of TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f. at 1732195585581Flushing 8328be8165f9f405996e915498f7765f/A: creating writer at 1732195585581Flushing 8328be8165f9f405996e915498f7765f/A: appending metadata at 1732195585584 (+3 ms)Flushing 8328be8165f9f405996e915498f7765f/A: closing flushed file at 1732195585584Flushing 8328be8165f9f405996e915498f7765f/B: creating writer at 1732195585591 (+7 ms)Flushing 8328be8165f9f405996e915498f7765f/B: appending metadata at 1732195585592 (+1 ms)Flushing 8328be8165f9f405996e915498f7765f/B: closing flushed file at 1732195585592Flushing 8328be8165f9f405996e915498f7765f/C: creating writer at 1732195585999 (+407 ms)Flushing 8328be8165f9f405996e915498f7765f/C: appending metadata at 1732195586001 (+2 ms)Flushing 8328be8165f9f405996e915498f7765f/C: closing flushed file at 1732195586001Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2fd1ad96: reopening flushed file at 1732195586409 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8a5917d: reopening flushed file at 1732195586412 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e77c65c: reopening flushed file at 1732195586415 (+3 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 8328be8165f9f405996e915498f7765f in 837ms, sequenceid=420, compaction requested=true at 1732195586419 (+4 ms)Writing region close event to WAL at 1732195586492 (+73 ms)Running coprocessor post-close hooks at 1732195586495 (+3 ms)Closed at 1732195586495 2024-11-21T13:26:26,497 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=144}] handler.UnassignRegionHandler(157): Closed 8328be8165f9f405996e915498f7765f 2024-11-21T13:26:26,497 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=8328be8165f9f405996e915498f7765f, regionState=CLOSED 2024-11-21T13:26:26,499 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; CloseRegionProcedure 8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:26:26,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-21T13:26:26,501 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; CloseRegionProcedure 8328be8165f9f405996e915498f7765f, server=4fe6efa1e611,33573,1732195430851 in 1.0710 sec 2024-11-21T13:26:26,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-21T13:26:26,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=8328be8165f9f405996e915498f7765f, UNASSIGN in 1.0750 sec 2024-11-21T13:26:26,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-21T13:26:26,504 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.0780 sec 2024-11-21T13:26:26,505 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195586504"}]},"ts":"1732195586504"} 2024-11-21T13:26:26,506 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T13:26:26,506 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T13:26:26,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.0850 sec 2024-11-21T13:26:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-21T13:26:26,560 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:26,560 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T13:26:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:26,562 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=145, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:26,562 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=145, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T13:26:26,564 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f 2024-11-21T13:26:26,566 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/recovered.edits] 2024-11-21T13:26:26,567 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/384f3d0fefde4e2184783fee19f4d334 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/384f3d0fefde4e2184783fee19f4d334 2024-11-21T13:26:26,568 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/386436acbd484303965058b9fbfbf3e4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/386436acbd484303965058b9fbfbf3e4 2024-11-21T13:26:26,569 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/a842b98868b64c8f95f1bf3971ccfd9c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/A/a842b98868b64c8f95f1bf3971ccfd9c 2024-11-21T13:26:26,570 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/1a318d57cd6440e5a90bc48a95e27c37 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/1a318d57cd6440e5a90bc48a95e27c37 2024-11-21T13:26:26,571 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/c4ceef2f28854f488f0dc844fe316375 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/c4ceef2f28854f488f0dc844fe316375 2024-11-21T13:26:26,571 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fe6c77cd19b94e019ea2210a67a48a74 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/B/fe6c77cd19b94e019ea2210a67a48a74 2024-11-21T13:26:26,573 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/3bfcdada3c834b79a8d1e4e434058deb to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/3bfcdada3c834b79a8d1e4e434058deb 2024-11-21T13:26:26,573 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/3caddc0098eb4f7595217b7fdb6f7701 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/3caddc0098eb4f7595217b7fdb6f7701 2024-11-21T13:26:26,574 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/bcb054efac57484aa1bc1acc0f1e0e30 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/C/bcb054efac57484aa1bc1acc0f1e0e30 2024-11-21T13:26:26,576 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/recovered.edits/423.seqid to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f/recovered.edits/423.seqid 2024-11-21T13:26:26,576 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/8328be8165f9f405996e915498f7765f 2024-11-21T13:26:26,576 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T13:26:26,578 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=145, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:26,580 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T13:26:26,582 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T13:26:26,583 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=145, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:26,583 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T13:26:26,583 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732195586583"}]},"ts":"9223372036854775807"} 2024-11-21T13:26:26,585 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T13:26:26,585 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 8328be8165f9f405996e915498f7765f, NAME => 'TestAcidGuarantees,,1732195552442.8328be8165f9f405996e915498f7765f.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T13:26:26,585 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T13:26:26,586 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732195586585"}]},"ts":"9223372036854775807"} 2024-11-21T13:26:26,587 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T13:26:26,588 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=145, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:26,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 28 msec 2024-11-21T13:26:26,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-21T13:26:26,670 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T13:26:26,670 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:26,678 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=236 (was 241), OpenFileDescriptor=441 (was 467), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=431 (was 511), ProcessCount=11 (was 11), AvailableMemoryMB=2087 (was 2294) 2024-11-21T13:26:26,686 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=236, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=431, ProcessCount=11, AvailableMemoryMB=2087 2024-11-21T13:26:26,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T13:26:26,687 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T13:26:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:26,689 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T13:26:26,689 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:26,689 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 146 2024-11-21T13:26:26,690 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T13:26:26,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T13:26:26,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742418_1594 (size=959) 2024-11-21T13:26:26,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T13:26:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T13:26:27,096 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48 2024-11-21T13:26:27,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742419_1595 (size=53) 2024-11-21T13:26:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T13:26:27,500 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:26:27,500 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 15530bc3c4bda91d823425a8a74e3630, disabling compactions & flushes 2024-11-21T13:26:27,500 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:27,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:27,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. after waiting 0 ms 2024-11-21T13:26:27,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:27,501 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:27,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 15530bc3c4bda91d823425a8a74e3630: Waiting for close lock at 1732195587500Disabling compacts and flushes for region at 1732195587500Disabling writes for close at 1732195587501 (+1 ms)Writing region close event to WAL at 1732195587501Closed at 1732195587501 2024-11-21T13:26:27,502 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T13:26:27,502 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732195587502"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732195587502"}]},"ts":"1732195587502"} 2024-11-21T13:26:27,503 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-21T13:26:27,504 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T13:26:27,504 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195587504"}]},"ts":"1732195587504"} 2024-11-21T13:26:27,505 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-21T13:26:27,505 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, ASSIGN}] 2024-11-21T13:26:27,506 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, ASSIGN 2024-11-21T13:26:27,507 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, ASSIGN; state=OFFLINE, location=4fe6efa1e611,33573,1732195430851; forceNewPlan=false, retain=false 2024-11-21T13:26:27,657 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=15530bc3c4bda91d823425a8a74e3630, regionState=OPENING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:27,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, ASSIGN because future has completed 2024-11-21T13:26:27,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:26:27,813 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:27,813 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7752): Opening region: {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} 2024-11-21T13:26:27,813 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:27,813 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:26:27,813 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7794): checking encryption for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:27,813 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(7797): checking classloading for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:27,814 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:27,815 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15530bc3c4bda91d823425a8a74e3630 columnFamilyName A 2024-11-21T13:26:27,815 DEBUG [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:27,816 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] regionserver.HStore(327): Store=15530bc3c4bda91d823425a8a74e3630/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:26:27,816 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:27,816 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15530bc3c4bda91d823425a8a74e3630 columnFamilyName B 2024-11-21T13:26:27,816 DEBUG [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:27,817 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] regionserver.HStore(327): Store=15530bc3c4bda91d823425a8a74e3630/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:26:27,817 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:27,817 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15530bc3c4bda91d823425a8a74e3630 columnFamilyName C 2024-11-21T13:26:27,817 DEBUG [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:27,818 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] regionserver.HStore(327): Store=15530bc3c4bda91d823425a8a74e3630/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:26:27,818 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1038): replaying wal for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:27,818 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:27,818 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:27,819 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1048): stopping wal replay for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:27,819 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1060): Cleaning up temporary data for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:27,819 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T13:26:27,820 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1093): writing seq id for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:27,822 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T13:26:27,822 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1114): Opened 15530bc3c4bda91d823425a8a74e3630; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61212333, jitterRate=-0.08786515891551971}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T13:26:27,822 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:27,823 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegion(1006): Region open journal for 15530bc3c4bda91d823425a8a74e3630: Running coprocessor pre-open hook at 1732195587813Writing region info on filesystem at 1732195587813Initializing all the Stores at 1732195587814 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195587814Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195587814Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195587814Cleaning up temporary data from old regions at 1732195587819 (+5 ms)Running coprocessor post-open hooks at 1732195587822 (+3 ms)Region opened successfully at 1732195587823 (+1 ms) 2024-11-21T13:26:27,823 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., pid=148, masterSystemTime=1732195587810 2024-11-21T13:26:27,825 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:27,825 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=148}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:27,825 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=15530bc3c4bda91d823425a8a74e3630, regionState=OPEN, openSeqNum=2, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:27,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:26:27,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-21T13:26:27,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure 15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 in 168 msec 2024-11-21T13:26:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T13:26:27,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-21T13:26:27,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, ASSIGN in 324 msec 2024-11-21T13:26:27,831 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T13:26:27,831 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195587831"}]},"ts":"1732195587831"} 2024-11-21T13:26:27,832 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-21T13:26:27,833 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=146, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T13:26:27,834 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-21T13:26:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-21T13:26:28,840 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:28,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a4bd4a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:28,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:26:28,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:26:28,842 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:26:28,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:26:28,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:26:28,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6ab33a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:28,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:26:28,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:26:28,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:28,843 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:26:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69163f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:28,843 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:26:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:26:28,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:26:28,845 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58334, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:26:28,845 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T13:26:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 4fe6efa1e611,46031,1732195430123 2024-11-21T13:26:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@646691c3 2024-11-21T13:26:28,846 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T13:26:28,847 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37730, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T13:26:28,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-21T13:26:28,848 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T13:26:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:28,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742420_1596 (size=995) 2024-11-21T13:26:29,258 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-21T13:26:29,258 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-21T13:26:29,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T13:26:29,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, REOPEN/MOVE}] 2024-11-21T13:26:29,263 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, REOPEN/MOVE 2024-11-21T13:26:29,263 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=15530bc3c4bda91d823425a8a74e3630, regionState=CLOSING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:29,264 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, REOPEN/MOVE because future has completed 2024-11-21T13:26:29,264 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T13:26:29,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:26:29,416 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,417 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T13:26:29,417 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 15530bc3c4bda91d823425a8a74e3630, disabling compactions & flushes 2024-11-21T13:26:29,417 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:29,417 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:29,417 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. after waiting 0 ms 2024-11-21T13:26:29,417 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:29,420 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T13:26:29,420 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:29,420 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 15530bc3c4bda91d823425a8a74e3630: Waiting for close lock at 1732195589417Running coprocessor pre-close hooks at 1732195589417Disabling compacts and flushes for region at 1732195589417Disabling writes for close at 1732195589417Writing region close event to WAL at 1732195589417Running coprocessor post-close hooks at 1732195589420 (+3 ms)Closed at 1732195589420 2024-11-21T13:26:29,421 WARN [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionServer(3299): Not adding moved region record: 15530bc3c4bda91d823425a8a74e3630 to self. 2024-11-21T13:26:29,422 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,423 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=15530bc3c4bda91d823425a8a74e3630, regionState=CLOSED 2024-11-21T13:26:29,424 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:26:29,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-21T13:26:29,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 in 160 msec 2024-11-21T13:26:29,426 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, REOPEN/MOVE; state=CLOSED, location=4fe6efa1e611,33573,1732195430851; forceNewPlan=false, retain=true 2024-11-21T13:26:29,577 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=15530bc3c4bda91d823425a8a74e3630, regionState=OPENING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:29,578 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, REOPEN/MOVE because future has completed 2024-11-21T13:26:29,578 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:26:29,732 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:29,732 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7752): Opening region: {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} 2024-11-21T13:26:29,733 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,733 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T13:26:29,733 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7794): checking encryption for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,733 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(7797): checking classloading for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,734 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,735 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15530bc3c4bda91d823425a8a74e3630 columnFamilyName A 2024-11-21T13:26:29,736 DEBUG [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:29,736 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] regionserver.HStore(327): Store=15530bc3c4bda91d823425a8a74e3630/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:26:29,736 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,737 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15530bc3c4bda91d823425a8a74e3630 columnFamilyName B 2024-11-21T13:26:29,737 DEBUG [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:29,737 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] regionserver.HStore(327): Store=15530bc3c4bda91d823425a8a74e3630/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:26:29,737 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,738 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 15530bc3c4bda91d823425a8a74e3630 columnFamilyName C 2024-11-21T13:26:29,738 DEBUG [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:29,738 INFO [StoreOpener-15530bc3c4bda91d823425a8a74e3630-1 {}] regionserver.HStore(327): Store=15530bc3c4bda91d823425a8a74e3630/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T13:26:29,738 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1038): replaying wal for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,739 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,739 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,740 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1048): stopping wal replay for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,740 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1060): Cleaning up temporary data for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,740 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-21T13:26:29,741 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1093): writing seq id for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,742 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1114): Opened 15530bc3c4bda91d823425a8a74e3630; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74106936, jitterRate=0.10427939891815186}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-21T13:26:29,742 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,742 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegion(1006): Region open journal for 15530bc3c4bda91d823425a8a74e3630: Running coprocessor pre-open hook at 1732195589733Writing region info on filesystem at 1732195589733Initializing all the Stores at 1732195589733Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195589734 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195589734Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732195589734Cleaning up temporary data from old regions at 1732195589740 (+6 ms)Running coprocessor post-open hooks at 1732195589742 (+2 ms)Region opened successfully at 1732195589742 2024-11-21T13:26:29,743 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., pid=153, masterSystemTime=1732195589730 2024-11-21T13:26:29,744 DEBUG [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:29,744 INFO [RS_OPEN_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_OPEN_REGION, pid=153}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:29,744 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=15530bc3c4bda91d823425a8a74e3630, regionState=OPEN, openSeqNum=5, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:29,745 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=151, state=RUNNABLE, hasLock=false; OpenRegionProcedure 15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:26:29,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=151 2024-11-21T13:26:29,747 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=151, state=SUCCESS, hasLock=false; OpenRegionProcedure 15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 in 168 msec 2024-11-21T13:26:29,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-21T13:26:29,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, REOPEN/MOVE in 485 msec 2024-11-21T13:26:29,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-21T13:26:29,751 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 489 msec 2024-11-21T13:26:29,753 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 903 msec 2024-11-21T13:26:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-21T13:26:29,754 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:29,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@616f2a91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:26:29,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:26:29,755 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:26:29,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:26:29,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:26:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6db75dec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:26:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:26:29,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:29,756 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:26:29,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@328e3aa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,757 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f6cc81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,757 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:26:29,757 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:26:29,758 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:26:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:26:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:26:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54d5ce9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:26:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:26:29,758 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:29,759 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:26:29,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4271cbc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cb0aa17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:26:29,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:26:29,760 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:26:29,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:26:29,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:26:29,760 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d308217, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:26:29,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:26:29,761 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:29,761 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:26:29,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2298676f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,762 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af17435, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,762 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:26:29,762 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:26:29,763 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:26:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:26:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:26:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@565d10a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:26:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:26:29,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:29,764 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37798, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:26:29,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@337a7fe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ad77ba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:26:29,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:26:29,765 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:26:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:26:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:26:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11d76c55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:26:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:26:29,766 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:29,766 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:26:29,767 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fbd32d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b8b3765, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:26:29,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:26:29,768 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:26:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:26:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:26:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@193e1279, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:26:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:26:29,768 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:29,769 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:26:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@558f38a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e7a0a59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:26:29,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:26:29,771 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:26:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:26:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:26:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b1f0b86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:26:29,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:26:29,772 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:29,772 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37842, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:26:29,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2693591e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,773 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5735b92e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,773 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:26:29,773 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:26:29,774 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:26:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:26:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:26:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66aae138, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:26:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:26:29,774 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:29,775 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:26:29,775 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b2d1504, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7388bb9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,776 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:26:29,776 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:26:29,777 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:26:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:26:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:26:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f207349, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:26:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:26:29,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:29,778 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:26:29,778 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5412342e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71d93d3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 4fe6efa1e611,46031,-1 for getting cluster id 2024-11-21T13:26:29,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T13:26:29,779 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'fe422165-ca34-4f60-b78c-7299a18fbab3' 2024-11-21T13:26:29,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T13:26:29,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "fe422165-ca34-4f60-b78c-7299a18fbab3" 2024-11-21T13:26:29,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cac7581, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [4fe6efa1e611,46031,-1] 2024-11-21T13:26:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T13:26:29,780 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:29,780 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37880, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T13:26:29,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@520d3d17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T13:26:29,781 DEBUG [Thread-2634 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:26:29,781 DEBUG [Thread-2638 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:26:29,781 DEBUG [Thread-2640 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:26:29,781 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:26:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:26:29,782 DEBUG [Thread-2636 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:26:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:26:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:26:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:26:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:26:29,782 DEBUG [Thread-2649 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:26:29,782 DEBUG [Thread-2651 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:26:29,783 DEBUG [Thread-2643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:26:29,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:26:29,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58364, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:26:29,783 DEBUG [Thread-2647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:26:29,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58350, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:26:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:26:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:26:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:26:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:26:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:26:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:26:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:26:29,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:26:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:26:29,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58380, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:26:29,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:26:29,784 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:26:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:26:29,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58366, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:26:29,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58394, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:26:29,785 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:26:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:26:29,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:26:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-21T13:26:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T13:26:29,786 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:26:29,786 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:29,787 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:29,787 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:26:29,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:26:29,788 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:26:29,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:26:29,792 DEBUG [Thread-2632 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:26:29,792 DEBUG [Thread-2645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T13:26:29,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:26:29,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=4fe6efa1e611,33573,1732195430851, seqNum=-1] 2024-11-21T13:26:29,793 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:26:29,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T13:26:29,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58416, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:26:29,794 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58426, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T13:26:29,795 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:26:29,795 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5] 2024-11-21T13:26:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:29,830 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:29,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-21T13:26:29,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216e4166981ee94ff4b088d33fa8863855_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195589829/Put/seqid=0 2024-11-21T13:26:29,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:29,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58350 deadline: 1732195649853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:29,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195649854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:29,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:29,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:29,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195649857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:29,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,861 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:29,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:29,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195649858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:29,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:29,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195649858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:29,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T13:26:29,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742421_1597 (size=12154) 2024-11-21T13:26:29,939 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T13:26:29,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:29,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:29,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:29,940 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:29,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:29,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:29,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58350 deadline: 1732195649969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195649969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:29,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195649969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:29,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195649969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195649970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:29,972 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:30,092 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T13:26:30,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:30,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:30,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:30,093 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T13:26:30,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:30,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58350 deadline: 1732195650179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:30,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195650179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:30,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195650179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195650179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:30,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195650180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,182 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:30,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T13:26:30,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:30,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:30,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:30,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,290 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:30,294 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216e4166981ee94ff4b088d33fa8863855_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216e4166981ee94ff4b088d33fa8863855_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:30,295 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/1f42cec327044ad8846a4560b62d1743, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:30,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/1f42cec327044ad8846a4560b62d1743 is 175, key is test_row_0/A:col10/1732195589829/Put/seqid=0 2024-11-21T13:26:30,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742422_1598 (size=30955) 2024-11-21T13:26:30,398 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T13:26:30,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:30,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:30,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:30,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T13:26:30,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195650489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:30,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:30,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58350 deadline: 1732195650489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:30,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195650489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,491 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:30,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195650490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,492 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195650490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:30,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:30,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T13:26:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:30,550 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,700 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/1f42cec327044ad8846a4560b62d1743 2024-11-21T13:26:30,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T13:26:30,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:30,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:30,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:30,704 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/a317c87798664ee78300042b1a70c1ca is 50, key is test_row_0/B:col10/1732195589829/Put/seqid=0 2024-11-21T13:26:30,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742423_1599 (size=12001) 2024-11-21T13:26:30,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/a317c87798664ee78300042b1a70c1ca 2024-11-21T13:26:30,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/f1960ec8999d4c559933343ebe2e236b is 50, key is test_row_0/C:col10/1732195589829/Put/seqid=0 2024-11-21T13:26:30,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742424_1600 (size=12001) 2024-11-21T13:26:30,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T13:26:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:30,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:30,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:30,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:30,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T13:26:31,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195650999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:31,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58350 deadline: 1732195650999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:31,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195650999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:31,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:31,001 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:31,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195650999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:31,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:31,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195650999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:31,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:31,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T13:26:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:31,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:31,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/f1960ec8999d4c559933343ebe2e236b 2024-11-21T13:26:31,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T13:26:31,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/1f42cec327044ad8846a4560b62d1743 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/1f42cec327044ad8846a4560b62d1743 2024-11-21T13:26:31,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:31,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:31,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:31,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:31,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:31,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:31,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/1f42cec327044ad8846a4560b62d1743, entries=150, sequenceid=15, filesize=30.2 K 2024-11-21T13:26:31,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/a317c87798664ee78300042b1a70c1ca as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a317c87798664ee78300042b1a70c1ca 2024-11-21T13:26:31,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a317c87798664ee78300042b1a70c1ca, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T13:26:31,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/f1960ec8999d4c559933343ebe2e236b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f1960ec8999d4c559933343ebe2e236b 2024-11-21T13:26:31,179 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f1960ec8999d4c559933343ebe2e236b, entries=150, sequenceid=15, filesize=11.7 K 2024-11-21T13:26:31,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 15530bc3c4bda91d823425a8a74e3630 in 1349ms, sequenceid=15, compaction requested=false 2024-11-21T13:26:31,179 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:31,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-21T13:26:31,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:31,314 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T13:26:31,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219bb264804b0e4162a20f346e9c61f603_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195589850/Put/seqid=0 2024-11-21T13:26:31,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742425_1601 (size=12154) 2024-11-21T13:26:31,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:31,745 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411219bb264804b0e4162a20f346e9c61f603_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219bb264804b0e4162a20f346e9c61f603_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:31,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/c623f805d6da4e3eb02a9e81e867ee93, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:31,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/c623f805d6da4e3eb02a9e81e867ee93 is 175, key is test_row_0/A:col10/1732195589850/Put/seqid=0 2024-11-21T13:26:31,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742426_1602 (size=30955) 2024-11-21T13:26:31,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T13:26:32,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:32,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:32,016 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T13:26:32,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195652017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195652018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195652019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58350 deadline: 1732195652019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,024 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,025 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195652026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195652129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,131 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195652130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195652130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195652130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,153 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/c623f805d6da4e3eb02a9e81e867ee93 2024-11-21T13:26:32,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/9342136bfdb4474396b9fe3377ac38c7 is 50, key is test_row_0/B:col10/1732195589850/Put/seqid=0 2024-11-21T13:26:32,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742427_1603 (size=12001) 2024-11-21T13:26:32,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195652338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195652339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,341 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195652339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,341 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195652339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,563 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/9342136bfdb4474396b9fe3377ac38c7 2024-11-21T13:26:32,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/b05115e213a94f6089341fbeb53038a9 is 50, key is test_row_0/C:col10/1732195589850/Put/seqid=0 2024-11-21T13:26:32,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742428_1604 (size=12001) 2024-11-21T13:26:32,596 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/b05115e213a94f6089341fbeb53038a9 2024-11-21T13:26:32,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/c623f805d6da4e3eb02a9e81e867ee93 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c623f805d6da4e3eb02a9e81e867ee93 2024-11-21T13:26:32,614 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c623f805d6da4e3eb02a9e81e867ee93, entries=150, sequenceid=40, filesize=30.2 K 2024-11-21T13:26:32,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/9342136bfdb4474396b9fe3377ac38c7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/9342136bfdb4474396b9fe3377ac38c7 2024-11-21T13:26:32,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,617 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/9342136bfdb4474396b9fe3377ac38c7, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T13:26:32,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/b05115e213a94f6089341fbeb53038a9 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/b05115e213a94f6089341fbeb53038a9 2024-11-21T13:26:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,621 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/b05115e213a94f6089341fbeb53038a9, entries=150, sequenceid=40, filesize=11.7 K 2024-11-21T13:26:32,621 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 15530bc3c4bda91d823425a8a74e3630 in 1307ms, sequenceid=40, compaction requested=false 2024-11-21T13:26:32,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:32,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:32,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-21T13:26:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-21T13:26:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-21T13:26:32,624 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8360 sec 2024-11-21T13:26:32,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.8440 sec 2024-11-21T13:26:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:32,657 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:32,657 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T13:26:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112152bddbb598e241ee882bd53fc807f84b_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_1/A:col10/1732195592656/Put/seqid=0 2024-11-21T13:26:32,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:32,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742429_1605 (size=12154) 2024-11-21T13:26:32,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195652693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,697 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195652695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195652696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195652697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195652798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195652809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195652809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,810 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:32,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:32,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195652809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:32,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:33,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:33,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195653009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:33,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195653019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:33,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195653019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:33,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195653019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,022 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:33,073 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:33,076 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112152bddbb598e241ee882bd53fc807f84b_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112152bddbb598e241ee882bd53fc807f84b_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:33,077 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/c00a7bae26534672abff176a6873d995, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:33,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/c00a7bae26534672abff176a6873d995 is 175, key is test_row_1/A:col10/1732195592656/Put/seqid=0 2024-11-21T13:26:33,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742430_1606 (size=30951) 2024-11-21T13:26:33,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:33,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195653319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:33,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:33,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:33,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195653329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:33,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195653329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,332 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:33,332 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:33,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195653329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:33,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,333 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:33,482 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/c00a7bae26534672abff176a6873d995 2024-11-21T13:26:33,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/f15c29d19e22495d86f5dd1c1d6b77e5 is 50, key is test_row_1/B:col10/1732195592656/Put/seqid=0 2024-11-21T13:26:33,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742431_1607 (size=9657) 2024-11-21T13:26:33,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:33,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195653829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:33,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195653839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195653839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:33,840 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:33,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195653839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:33,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:33,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/f15c29d19e22495d86f5dd1c1d6b77e5 2024-11-21T13:26:33,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/88804529cc514680ba0f400deca7a7aa is 50, key is test_row_1/C:col10/1732195592656/Put/seqid=0 2024-11-21T13:26:33,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742432_1608 (size=9657) 2024-11-21T13:26:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-21T13:26:33,939 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:33,940 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-21T13:26:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T13:26:33,941 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:33,942 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:33,942 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:34,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T13:26:34,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:34,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58350 deadline: 1732195654049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:34,051 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:34,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:34,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T13:26:34,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:34,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:34,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:34,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:34,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:34,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T13:26:34,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:34,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:34,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:34,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:34,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:34,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:34,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T13:26:34,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/88804529cc514680ba0f400deca7a7aa 2024-11-21T13:26:34,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/c00a7bae26534672abff176a6873d995 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c00a7bae26534672abff176a6873d995 2024-11-21T13:26:34,331 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c00a7bae26534672abff176a6873d995, entries=150, sequenceid=52, filesize=30.2 K 2024-11-21T13:26:34,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/f15c29d19e22495d86f5dd1c1d6b77e5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/f15c29d19e22495d86f5dd1c1d6b77e5 2024-11-21T13:26:34,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/f15c29d19e22495d86f5dd1c1d6b77e5, entries=100, sequenceid=52, filesize=9.4 K 2024-11-21T13:26:34,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/88804529cc514680ba0f400deca7a7aa as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/88804529cc514680ba0f400deca7a7aa 2024-11-21T13:26:34,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/88804529cc514680ba0f400deca7a7aa, entries=100, sequenceid=52, filesize=9.4 K 2024-11-21T13:26:34,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 15530bc3c4bda91d823425a8a74e3630 in 1681ms, sequenceid=52, compaction requested=true 2024-11-21T13:26:34,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:34,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:34,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:34,339 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:34,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:34,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:34,339 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:34,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:34,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:34,339 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:34,339 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92861 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:34,339 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/B is initiating minor compaction (all files) 2024-11-21T13:26:34,339 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/A is initiating minor compaction (all files) 2024-11-21T13:26:34,340 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/B in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:34,340 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/A in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:34,340 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/1f42cec327044ad8846a4560b62d1743, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c623f805d6da4e3eb02a9e81e867ee93, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c00a7bae26534672abff176a6873d995] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=90.7 K 2024-11-21T13:26:34,340 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a317c87798664ee78300042b1a70c1ca, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/9342136bfdb4474396b9fe3377ac38c7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/f15c29d19e22495d86f5dd1c1d6b77e5] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=32.9 K 2024-11-21T13:26:34,340 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:34,340 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/1f42cec327044ad8846a4560b62d1743, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c623f805d6da4e3eb02a9e81e867ee93, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c00a7bae26534672abff176a6873d995] 2024-11-21T13:26:34,340 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting a317c87798664ee78300042b1a70c1ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732195589828 2024-11-21T13:26:34,340 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f42cec327044ad8846a4560b62d1743, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732195589828 2024-11-21T13:26:34,340 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9342136bfdb4474396b9fe3377ac38c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732195589850 2024-11-21T13:26:34,340 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting c623f805d6da4e3eb02a9e81e867ee93, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732195589850 2024-11-21T13:26:34,340 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting c00a7bae26534672abff176a6873d995, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732195592016 2024-11-21T13:26:34,341 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f15c29d19e22495d86f5dd1c1d6b77e5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732195592016 2024-11-21T13:26:34,350 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:34,359 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411217e16e9227f9446b1885b8cbc7dec06be_15530bc3c4bda91d823425a8a74e3630 store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:34,360 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#B#compaction#513 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:34,360 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411217e16e9227f9446b1885b8cbc7dec06be_15530bc3c4bda91d823425a8a74e3630, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:34,360 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411217e16e9227f9446b1885b8cbc7dec06be_15530bc3c4bda91d823425a8a74e3630 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:34,360 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/c5bdfea323c441ec91f754d8f9148dc8 is 50, key is test_row_0/B:col10/1732195589850/Put/seqid=0 2024-11-21T13:26:34,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742434_1610 (size=4469) 2024-11-21T13:26:34,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742433_1609 (size=12104) 2024-11-21T13:26:34,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-21T13:26:34,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:34,400 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-21T13:26:34,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121030b64f5b2a34db8bdcd96c213c2c6f7_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195592690/Put/seqid=0 2024-11-21T13:26:34,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742435_1611 (size=12154) 2024-11-21T13:26:34,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T13:26:34,782 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#A#compaction#512 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:34,783 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/db16f4e7772d4cb29d43ca5985667582 is 175, key is test_row_0/A:col10/1732195589850/Put/seqid=0 2024-11-21T13:26:34,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742436_1612 (size=31165) 2024-11-21T13:26:34,787 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/c5bdfea323c441ec91f754d8f9148dc8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/c5bdfea323c441ec91f754d8f9148dc8 2024-11-21T13:26:34,791 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/db16f4e7772d4cb29d43ca5985667582 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/db16f4e7772d4cb29d43ca5985667582 2024-11-21T13:26:34,793 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/B of 15530bc3c4bda91d823425a8a74e3630 into c5bdfea323c441ec91f754d8f9148dc8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:34,793 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:34,793 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/B, priority=13, startTime=1732195594339; duration=0sec 2024-11-21T13:26:34,793 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:34,793 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:B 2024-11-21T13:26:34,793 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:34,794 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:34,794 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/C is initiating minor compaction (all files) 2024-11-21T13:26:34,794 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/C in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:34,794 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f1960ec8999d4c559933343ebe2e236b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/b05115e213a94f6089341fbeb53038a9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/88804529cc514680ba0f400deca7a7aa] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=32.9 K 2024-11-21T13:26:34,794 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f1960ec8999d4c559933343ebe2e236b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732195589828 2024-11-21T13:26:34,795 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting b05115e213a94f6089341fbeb53038a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732195589850 2024-11-21T13:26:34,796 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 88804529cc514680ba0f400deca7a7aa, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732195592016 2024-11-21T13:26:34,796 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/A of 15530bc3c4bda91d823425a8a74e3630 into db16f4e7772d4cb29d43ca5985667582(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:34,796 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:34,796 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/A, priority=13, startTime=1732195594339; duration=0sec 2024-11-21T13:26:34,796 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:34,796 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:A 2024-11-21T13:26:34,810 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#C#compaction#515 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:34,810 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/5cd0d79889444361a4bb52104becd9a7 is 50, key is test_row_0/C:col10/1732195589850/Put/seqid=0 2024-11-21T13:26:34,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:34,817 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121030b64f5b2a34db8bdcd96c213c2c6f7_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121030b64f5b2a34db8bdcd96c213c2c6f7_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:34,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/68e1cdec24de4470a5b8cb3e7a56d9cd, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:34,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/68e1cdec24de4470a5b8cb3e7a56d9cd is 175, key is test_row_0/A:col10/1732195592690/Put/seqid=0 2024-11-21T13:26:34,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742437_1613 (size=12104) 2024-11-21T13:26:34,833 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/5cd0d79889444361a4bb52104becd9a7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/5cd0d79889444361a4bb52104becd9a7 2024-11-21T13:26:34,837 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/C of 15530bc3c4bda91d823425a8a74e3630 into 5cd0d79889444361a4bb52104becd9a7(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:34,837 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:34,837 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/C, priority=13, startTime=1732195594339; duration=0sec 2024-11-21T13:26:34,838 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:34,838 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:C 2024-11-21T13:26:34,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742438_1614 (size=30955) 2024-11-21T13:26:34,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:34,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195654853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:34,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195654855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:34,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:34,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195654855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:34,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:34,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:34,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195654859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:34,861 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,862 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:34,961 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:34,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195654959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:34,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:34,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195654969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195654969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:34,972 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:35,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T13:26:35,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:35,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195655169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:35,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:35,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195655179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:35,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195655179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:35,239 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/68e1cdec24de4470a5b8cb3e7a56d9cd 2024-11-21T13:26:35,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/1692c4f2420d4b2b817da5cd677eed12 is 50, key is test_row_0/B:col10/1732195592690/Put/seqid=0 2024-11-21T13:26:35,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742439_1615 (size=12001) 2024-11-21T13:26:35,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195655479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:35,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195655489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:35,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:35,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195655489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:35,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,491 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:35,650 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/1692c4f2420d4b2b817da5cd677eed12 2024-11-21T13:26:35,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/ffdf4abdc4224bdda017b573f3fc0300 is 50, key is test_row_0/C:col10/1732195592690/Put/seqid=0 2024-11-21T13:26:35,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742440_1616 (size=12001) 2024-11-21T13:26:35,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195655989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:35,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:35,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:36,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195655999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:36,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:36,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:36,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:36,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:36,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195655999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:36,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:36,070 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/ffdf4abdc4224bdda017b573f3fc0300 2024-11-21T13:26:36,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/68e1cdec24de4470a5b8cb3e7a56d9cd as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/68e1cdec24de4470a5b8cb3e7a56d9cd 2024-11-21T13:26:36,077 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/68e1cdec24de4470a5b8cb3e7a56d9cd, entries=150, sequenceid=77, filesize=30.2 K 2024-11-21T13:26:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/1692c4f2420d4b2b817da5cd677eed12 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/1692c4f2420d4b2b817da5cd677eed12 2024-11-21T13:26:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,087 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/1692c4f2420d4b2b817da5cd677eed12, entries=150, sequenceid=77, filesize=11.7 K 2024-11-21T13:26:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/ffdf4abdc4224bdda017b573f3fc0300 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/ffdf4abdc4224bdda017b573f3fc0300 2024-11-21T13:26:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T13:26:36,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,091 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/ffdf4abdc4224bdda017b573f3fc0300, entries=150, sequenceid=77, filesize=11.7 K 2024-11-21T13:26:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,091 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 15530bc3c4bda91d823425a8a74e3630 in 1691ms, sequenceid=77, compaction requested=false 2024-11-21T13:26:36,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:36,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:36,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-21T13:26:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-21T13:26:36,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-21T13:26:36,094 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1510 sec 2024-11-21T13:26:36,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.1550 sec 2024-11-21T13:26:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:36,883 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:36,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,883 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-21T13:26:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e3edb5b64d434059a87bd126cd693292_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195596882/Put/seqid=0 2024-11-21T13:26:36,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:36,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742441_1617 (size=12154) 2024-11-21T13:26:36,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:36,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195656975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:37,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:37,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195656999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:37,002 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_1' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:37,003 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:37,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195657009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:37,013 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:37,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:37,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:37,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:37,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195657019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:37,023 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:37,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:37,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:37,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195657079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:37,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:37,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:37,082 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:37,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195657289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:37,300 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:37,304 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e3edb5b64d434059a87bd126cd693292_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e3edb5b64d434059a87bd126cd693292_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:37,305 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/d3172bf5ee98432cac99fdb93ce263d5, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:37,306 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/d3172bf5ee98432cac99fdb93ce263d5 is 175, key is test_row_0/A:col10/1732195596882/Put/seqid=0 2024-11-21T13:26:37,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742442_1618 (size=30951) 2024-11-21T13:26:37,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195657599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:37,737 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/d3172bf5ee98432cac99fdb93ce263d5 2024-11-21T13:26:37,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/44eb52f65350443ebcc5507be30aae22 is 50, key is test_row_0/B:col10/1732195596882/Put/seqid=0 2024-11-21T13:26:37,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742443_1619 (size=9657) 2024-11-21T13:26:37,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/44eb52f65350443ebcc5507be30aae22 2024-11-21T13:26:37,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/9e962c0cc32643e69342c9e22c62e879 is 50, key is test_row_0/C:col10/1732195596882/Put/seqid=0 2024-11-21T13:26:37,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742444_1620 (size=9657) 2024-11-21T13:26:38,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:38,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58350 deadline: 1732195658069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:38,070 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:38,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:38,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-21T13:26:38,109 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:38,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-21T13:26:38,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T13:26:38,111 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:38,112 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:38,112 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:38,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:38,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195658109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:38,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:38,113 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:38,164 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/9e962c0cc32643e69342c9e22c62e879 2024-11-21T13:26:38,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/d3172bf5ee98432cac99fdb93ce263d5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d3172bf5ee98432cac99fdb93ce263d5 2024-11-21T13:26:38,177 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d3172bf5ee98432cac99fdb93ce263d5, entries=150, sequenceid=92, filesize=30.2 K 2024-11-21T13:26:38,178 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/44eb52f65350443ebcc5507be30aae22 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/44eb52f65350443ebcc5507be30aae22 2024-11-21T13:26:38,181 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/44eb52f65350443ebcc5507be30aae22, entries=100, sequenceid=92, filesize=9.4 K 2024-11-21T13:26:38,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/9e962c0cc32643e69342c9e22c62e879 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/9e962c0cc32643e69342c9e22c62e879 2024-11-21T13:26:38,185 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/9e962c0cc32643e69342c9e22c62e879, entries=100, sequenceid=92, filesize=9.4 K 2024-11-21T13:26:38,186 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-21T13:26:38,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 15530bc3c4bda91d823425a8a74e3630 in 1304ms, sequenceid=92, compaction requested=true 2024-11-21T13:26:38,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:38,187 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:38,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:38,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:38,188 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:38,188 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93071 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:38,188 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/A is initiating minor compaction (all files) 2024-11-21T13:26:38,188 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/A in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:38,188 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/db16f4e7772d4cb29d43ca5985667582, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/68e1cdec24de4470a5b8cb3e7a56d9cd, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d3172bf5ee98432cac99fdb93ce263d5] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=90.9 K 2024-11-21T13:26:38,188 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:38,188 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/db16f4e7772d4cb29d43ca5985667582, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/68e1cdec24de4470a5b8cb3e7a56d9cd, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d3172bf5ee98432cac99fdb93ce263d5] 2024-11-21T13:26:38,189 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting db16f4e7772d4cb29d43ca5985667582, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732195589850 2024-11-21T13:26:38,189 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:38,189 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/B is initiating minor compaction (all files) 2024-11-21T13:26:38,189 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/B in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:38,189 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/c5bdfea323c441ec91f754d8f9148dc8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/1692c4f2420d4b2b817da5cd677eed12, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/44eb52f65350443ebcc5507be30aae22] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=33.0 K 2024-11-21T13:26:38,189 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68e1cdec24de4470a5b8cb3e7a56d9cd, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732195592690 2024-11-21T13:26:38,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:38,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:38,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:38,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:38,193 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3172bf5ee98432cac99fdb93ce263d5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732195594854 2024-11-21T13:26:38,193 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting c5bdfea323c441ec91f754d8f9148dc8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732195589850 2024-11-21T13:26:38,200 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 1692c4f2420d4b2b817da5cd677eed12, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732195592690 2024-11-21T13:26:38,201 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 44eb52f65350443ebcc5507be30aae22, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732195594854 2024-11-21T13:26:38,208 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#B#compaction#521 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:38,208 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/29de47f09e52450ca78fe6ab3837dac1 is 50, key is test_row_0/B:col10/1732195596882/Put/seqid=0 2024-11-21T13:26:38,211 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:38,213 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121b62b8d2bcaba462fb6c82178a3c937d7_15530bc3c4bda91d823425a8a74e3630 store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:38,214 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121b62b8d2bcaba462fb6c82178a3c937d7_15530bc3c4bda91d823425a8a74e3630, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:38,214 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121b62b8d2bcaba462fb6c82178a3c937d7_15530bc3c4bda91d823425a8a74e3630 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:38,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T13:26:38,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742446_1622 (size=4469) 2024-11-21T13:26:38,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742445_1621 (size=12207) 2024-11-21T13:26:38,247 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/29de47f09e52450ca78fe6ab3837dac1 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/29de47f09e52450ca78fe6ab3837dac1 2024-11-21T13:26:38,251 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/B of 15530bc3c4bda91d823425a8a74e3630 into 29de47f09e52450ca78fe6ab3837dac1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:38,251 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:38,252 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/B, priority=13, startTime=1732195598188; duration=0sec 2024-11-21T13:26:38,252 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:38,252 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:B 2024-11-21T13:26:38,252 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:38,253 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:38,253 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/C is initiating minor compaction (all files) 2024-11-21T13:26:38,253 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/C in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:38,253 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/5cd0d79889444361a4bb52104becd9a7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/ffdf4abdc4224bdda017b573f3fc0300, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/9e962c0cc32643e69342c9e22c62e879] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=33.0 K 2024-11-21T13:26:38,253 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd0d79889444361a4bb52104becd9a7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732195589850 2024-11-21T13:26:38,253 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting ffdf4abdc4224bdda017b573f3fc0300, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1732195592690 2024-11-21T13:26:38,254 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e962c0cc32643e69342c9e22c62e879, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732195594854 2024-11-21T13:26:38,260 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#C#compaction#523 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:38,261 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/2edfc464bf16498ea616ecd7eea1143e is 50, key is test_row_0/C:col10/1732195596882/Put/seqid=0 2024-11-21T13:26:38,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-21T13:26:38,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:38,265 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-21T13:26:38,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742447_1623 (size=12207) 2024-11-21T13:26:38,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121786bf86a3bbb4a3a8431f2f0c7908ee0_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195596964/Put/seqid=0 2024-11-21T13:26:38,280 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/2edfc464bf16498ea616ecd7eea1143e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/2edfc464bf16498ea616ecd7eea1143e 2024-11-21T13:26:38,285 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/C of 15530bc3c4bda91d823425a8a74e3630 into 2edfc464bf16498ea616ecd7eea1143e(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:38,285 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:38,286 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/C, priority=13, startTime=1732195598189; duration=0sec 2024-11-21T13:26:38,286 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:38,286 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:C 2024-11-21T13:26:38,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742448_1624 (size=12154) 2024-11-21T13:26:38,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:38,320 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121786bf86a3bbb4a3a8431f2f0c7908ee0_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121786bf86a3bbb4a3a8431f2f0c7908ee0_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:38,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/bd94f0b39ce3498ab35506689666f76e, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:38,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/bd94f0b39ce3498ab35506689666f76e is 175, key is test_row_0/A:col10/1732195596964/Put/seqid=0 2024-11-21T13:26:38,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742449_1625 (size=30955) 2024-11-21T13:26:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T13:26:38,627 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#A#compaction#522 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:38,627 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/d5deddeb715f45ac93c554cf1b301d5c is 175, key is test_row_0/A:col10/1732195596882/Put/seqid=0 2024-11-21T13:26:38,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742450_1626 (size=31268) 2024-11-21T13:26:38,641 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/d5deddeb715f45ac93c554cf1b301d5c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d5deddeb715f45ac93c554cf1b301d5c 2024-11-21T13:26:38,646 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/A of 15530bc3c4bda91d823425a8a74e3630 into d5deddeb715f45ac93c554cf1b301d5c(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:38,646 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:38,646 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/A, priority=13, startTime=1732195598187; duration=0sec 2024-11-21T13:26:38,646 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:38,646 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:A 2024-11-21T13:26:38,735 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/bd94f0b39ce3498ab35506689666f76e 2024-11-21T13:26:38,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T13:26:38,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/e55ea4cb29d54b1a912fbf40cb449a82 is 50, key is test_row_0/B:col10/1732195596964/Put/seqid=0 2024-11-21T13:26:38,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742451_1627 (size=12001) 2024-11-21T13:26:38,745 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/e55ea4cb29d54b1a912fbf40cb449a82 2024-11-21T13:26:38,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/0648d2bae07b4222871e32ff436d79ff is 50, key is test_row_0/C:col10/1732195596964/Put/seqid=0 2024-11-21T13:26:38,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742452_1628 (size=12001) 2024-11-21T13:26:39,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:39,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:39,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:39,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195659034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:39,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:39,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195659034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:39,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195659039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:39,041 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:39,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195659119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:39,120 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2145 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:39,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:39,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195659149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:39,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195659149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:39,159 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/0648d2bae07b4222871e32ff436d79ff 2024-11-21T13:26:39,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/bd94f0b39ce3498ab35506689666f76e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/bd94f0b39ce3498ab35506689666f76e 2024-11-21T13:26:39,166 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/bd94f0b39ce3498ab35506689666f76e, entries=150, sequenceid=117, filesize=30.2 K 2024-11-21T13:26:39,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/e55ea4cb29d54b1a912fbf40cb449a82 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/e55ea4cb29d54b1a912fbf40cb449a82 2024-11-21T13:26:39,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,169 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/e55ea4cb29d54b1a912fbf40cb449a82, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T13:26:39,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/0648d2bae07b4222871e32ff436d79ff as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/0648d2bae07b4222871e32ff436d79ff 2024-11-21T13:26:39,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,174 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/0648d2bae07b4222871e32ff436d79ff, entries=150, sequenceid=117, filesize=11.7 K 2024-11-21T13:26:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,175 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 15530bc3c4bda91d823425a8a74e3630 in 910ms, sequenceid=117, compaction requested=false 2024-11-21T13:26:39,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:39,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:39,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-21T13:26:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-21T13:26:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-21T13:26:39,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0670 sec 2024-11-21T13:26:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,183 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 1.0710 sec 2024-11-21T13:26:39,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-21T13:26:39,249 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:39,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:39,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-21T13:26:39,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T13:26:39,252 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:39,253 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:39,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:39,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T13:26:39,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:39,362 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:39,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-21T13:26:39,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e5dfa0ac3750408c93807a7cec1aeed7_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195599033/Put/seqid=0 2024-11-21T13:26:39,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742453_1629 (size=12254) 2024-11-21T13:26:39,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T13:26:39,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:39,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:39,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:39,405 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:39,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:39,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:39,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:39,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195659413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:39,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195659415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:39,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:39,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195659519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:39,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:39,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195659529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,531 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:39,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T13:26:39,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:39,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:39,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:39,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:39,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:39,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T13:26:39,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T13:26:39,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:39,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:39,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:39,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:39,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:39,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:39,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:39,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195659728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:39,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:39,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:39,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195659739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:39,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:39,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:39,783 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:39,786 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e5dfa0ac3750408c93807a7cec1aeed7_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e5dfa0ac3750408c93807a7cec1aeed7_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:39,786 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/0d04edc744554e589d032576327d5a35, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:39,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/0d04edc744554e589d032576327d5a35 is 175, key is test_row_0/A:col10/1732195599033/Put/seqid=0 2024-11-21T13:26:39,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742454_1630 (size=31051) 2024-11-21T13:26:39,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T13:26:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:39,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:39,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:39,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:39,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T13:26:40,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T13:26:40,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:40,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:40,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195660039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:40,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:40,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195660049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:40,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:40,191 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/0d04edc744554e589d032576327d5a35 2024-11-21T13:26:40,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T13:26:40,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:40,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,192 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/a618ea62b098400b82b107791bfd6caf is 50, key is test_row_0/B:col10/1732195599033/Put/seqid=0 2024-11-21T13:26:40,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742455_1631 (size=9757) 2024-11-21T13:26:40,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T13:26:40,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:40,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,345 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T13:26:40,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T13:26:40,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:40,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,497 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:40,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195660549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:40,553 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:40,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195660559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:40,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:40,601 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/a618ea62b098400b82b107791bfd6caf 2024-11-21T13:26:40,606 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/f2d97b0a855a4af6af04c3630f77016d is 50, key is test_row_0/C:col10/1732195599033/Put/seqid=0 2024-11-21T13:26:40,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742456_1632 (size=9757) 2024-11-21T13:26:40,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T13:26:40,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:40,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,650 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,802 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T13:26:40,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:40,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,802 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,802 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T13:26:40,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:40,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:40,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:40,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:41,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/f2d97b0a855a4af6af04c3630f77016d 2024-11-21T13:26:41,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/0d04edc744554e589d032576327d5a35 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/0d04edc744554e589d032576327d5a35 2024-11-21T13:26:41,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/0d04edc744554e589d032576327d5a35, entries=150, sequenceid=133, filesize=30.3 K 2024-11-21T13:26:41,017 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/a618ea62b098400b82b107791bfd6caf as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a618ea62b098400b82b107791bfd6caf 2024-11-21T13:26:41,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a618ea62b098400b82b107791bfd6caf, entries=100, sequenceid=133, filesize=9.5 K 2024-11-21T13:26:41,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/f2d97b0a855a4af6af04c3630f77016d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f2d97b0a855a4af6af04c3630f77016d 2024-11-21T13:26:41,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f2d97b0a855a4af6af04c3630f77016d, entries=100, sequenceid=133, filesize=9.5 K 2024-11-21T13:26:41,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 15530bc3c4bda91d823425a8a74e3630 in 1661ms, sequenceid=133, compaction requested=true 2024-11-21T13:26:41,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:41,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:41,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:41,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:41,023 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:41,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:41,023 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:41,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:41,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:41,024 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:41,024 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93274 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:41,024 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/B is initiating minor compaction (all files) 2024-11-21T13:26:41,024 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/A is initiating minor compaction (all files) 2024-11-21T13:26:41,024 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/B in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:41,024 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/A in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:41,024 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/29de47f09e52450ca78fe6ab3837dac1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/e55ea4cb29d54b1a912fbf40cb449a82, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a618ea62b098400b82b107791bfd6caf] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=33.2 K 2024-11-21T13:26:41,024 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d5deddeb715f45ac93c554cf1b301d5c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/bd94f0b39ce3498ab35506689666f76e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/0d04edc744554e589d032576327d5a35] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=91.1 K 2024-11-21T13:26:41,024 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:41,024 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d5deddeb715f45ac93c554cf1b301d5c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/bd94f0b39ce3498ab35506689666f76e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/0d04edc744554e589d032576327d5a35] 2024-11-21T13:26:41,024 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 29de47f09e52450ca78fe6ab3837dac1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732195592693 2024-11-21T13:26:41,025 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5deddeb715f45ac93c554cf1b301d5c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732195592693 2024-11-21T13:26:41,025 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e55ea4cb29d54b1a912fbf40cb449a82, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732195596964 2024-11-21T13:26:41,025 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd94f0b39ce3498ab35506689666f76e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732195596964 2024-11-21T13:26:41,026 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d04edc744554e589d032576327d5a35, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732195599025 2024-11-21T13:26:41,026 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting a618ea62b098400b82b107791bfd6caf, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732195599033 2024-11-21T13:26:41,031 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:41,033 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121972ab053718a4f01a9e07725a8f188ae_15530bc3c4bda91d823425a8a74e3630 store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:41,033 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#B#compaction#531 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:41,033 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/8816cac0afa0476fa4cd5bc1e45cd73c is 50, key is test_row_0/B:col10/1732195599033/Put/seqid=0 2024-11-21T13:26:41,034 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121972ab053718a4f01a9e07725a8f188ae_15530bc3c4bda91d823425a8a74e3630, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:41,034 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121972ab053718a4f01a9e07725a8f188ae_15530bc3c4bda91d823425a8a74e3630 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:41,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742457_1633 (size=12409) 2024-11-21T13:26:41,059 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/8816cac0afa0476fa4cd5bc1e45cd73c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/8816cac0afa0476fa4cd5bc1e45cd73c 2024-11-21T13:26:41,063 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/B of 15530bc3c4bda91d823425a8a74e3630 into 8816cac0afa0476fa4cd5bc1e45cd73c(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:41,063 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:41,063 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/B, priority=13, startTime=1732195601023; duration=0sec 2024-11-21T13:26:41,063 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:41,063 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:B 2024-11-21T13:26:41,063 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:41,064 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:41,064 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/C is initiating minor compaction (all files) 2024-11-21T13:26:41,064 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/C in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:41,064 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/2edfc464bf16498ea616ecd7eea1143e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/0648d2bae07b4222871e32ff436d79ff, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f2d97b0a855a4af6af04c3630f77016d] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=33.2 K 2024-11-21T13:26:41,064 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 2edfc464bf16498ea616ecd7eea1143e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1732195592693 2024-11-21T13:26:41,064 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 0648d2bae07b4222871e32ff436d79ff, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732195596964 2024-11-21T13:26:41,065 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f2d97b0a855a4af6af04c3630f77016d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732195599033 2024-11-21T13:26:41,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742458_1634 (size=4469) 2024-11-21T13:26:41,075 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#A#compaction#530 average throughput is 0.56 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:41,076 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/8b3f315041974380be04746fb9680227 is 175, key is test_row_0/A:col10/1732195599033/Put/seqid=0 2024-11-21T13:26:41,082 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#C#compaction#532 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:41,082 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/792cd9f9289e40c2b70f8b8fb6efce8a is 50, key is test_row_0/C:col10/1732195599033/Put/seqid=0 2024-11-21T13:26:41,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-21T13:26:41,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:41,108 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-21T13:26:41,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742460_1636 (size=12409) 2024-11-21T13:26:41,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742459_1635 (size=31470) 2024-11-21T13:26:41,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210c1351d55adf4cc2a86afea3c84dc768_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195599413/Put/seqid=0 2024-11-21T13:26:41,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742461_1637 (size=12304) 2024-11-21T13:26:41,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:41,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:41,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195661178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:41,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:41,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195661289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:41,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:41,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:41,293 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T13:26:41,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:41,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195661499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:41,500 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:41,501 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:41,518 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/8b3f315041974380be04746fb9680227 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8b3f315041974380be04746fb9680227 2024-11-21T13:26:41,518 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/792cd9f9289e40c2b70f8b8fb6efce8a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/792cd9f9289e40c2b70f8b8fb6efce8a 2024-11-21T13:26:41,528 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/C of 15530bc3c4bda91d823425a8a74e3630 into 792cd9f9289e40c2b70f8b8fb6efce8a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:41,528 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/A of 15530bc3c4bda91d823425a8a74e3630 into 8b3f315041974380be04746fb9680227(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:41,528 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:41,528 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:41,528 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/A, priority=13, startTime=1732195601023; duration=0sec 2024-11-21T13:26:41,528 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/C, priority=13, startTime=1732195601023; duration=0sec 2024-11-21T13:26:41,528 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:41,528 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:41,528 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:C 2024-11-21T13:26:41,528 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:A 2024-11-21T13:26:41,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:41,534 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411210c1351d55adf4cc2a86afea3c84dc768_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210c1351d55adf4cc2a86afea3c84dc768_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:41,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/8523ceb9006847ee93d560c68dc31d5b, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:41,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/8523ceb9006847ee93d560c68dc31d5b is 175, key is test_row_0/A:col10/1732195599413/Put/seqid=0 2024-11-21T13:26:41,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742462_1638 (size=31105) 2024-11-21T13:26:41,538 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/8523ceb9006847ee93d560c68dc31d5b 2024-11-21T13:26:41,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/04933d7b7f874088b93c76771e8e675e is 50, key is test_row_0/B:col10/1732195599413/Put/seqid=0 2024-11-21T13:26:41,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742463_1639 (size=12151) 2024-11-21T13:26:41,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:41,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195661559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:41,560 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2147 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:41,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:41,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195661579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:41,580 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:41,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:41,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:41,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195661809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:41,811 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:41,950 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/04933d7b7f874088b93c76771e8e675e 2024-11-21T13:26:41,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/32b7bbf2d9a7426ab025f441bdb94ac9 is 50, key is test_row_0/C:col10/1732195599413/Put/seqid=0 2024-11-21T13:26:41,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742464_1640 (size=12151) 2024-11-21T13:26:42,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:42,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195662320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:42,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:42,374 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/32b7bbf2d9a7426ab025f441bdb94ac9 2024-11-21T13:26:42,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/8523ceb9006847ee93d560c68dc31d5b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8523ceb9006847ee93d560c68dc31d5b 2024-11-21T13:26:42,381 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8523ceb9006847ee93d560c68dc31d5b, entries=150, sequenceid=156, filesize=30.4 K 2024-11-21T13:26:42,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/04933d7b7f874088b93c76771e8e675e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/04933d7b7f874088b93c76771e8e675e 2024-11-21T13:26:42,384 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/04933d7b7f874088b93c76771e8e675e, entries=150, sequenceid=156, filesize=11.9 K 2024-11-21T13:26:42,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/32b7bbf2d9a7426ab025f441bdb94ac9 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/32b7bbf2d9a7426ab025f441bdb94ac9 2024-11-21T13:26:42,392 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/32b7bbf2d9a7426ab025f441bdb94ac9, entries=150, sequenceid=156, filesize=11.9 K 2024-11-21T13:26:42,393 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 15530bc3c4bda91d823425a8a74e3630 in 1285ms, sequenceid=156, compaction requested=false 2024-11-21T13:26:42,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:42,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:42,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-21T13:26:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-21T13:26:42,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-21T13:26:42,396 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1410 sec 2024-11-21T13:26:42,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 3.1470 sec 2024-11-21T13:26:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:43,080 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:43,081 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-21T13:26:43,087 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cd0333e5564e40ef91cf2e6176722223_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195601162/Put/seqid=0 2024-11-21T13:26:43,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742465_1641 (size=14794) 2024-11-21T13:26:43,106 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:43,109 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121cd0333e5564e40ef91cf2e6176722223_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cd0333e5564e40ef91cf2e6176722223_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:43,109 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/3b98a86e7b0445d0861a39abfd60f058, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:43,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/3b98a86e7b0445d0861a39abfd60f058 is 175, key is test_row_0/A:col10/1732195601162/Put/seqid=0 2024-11-21T13:26:43,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742466_1642 (size=39749) 2024-11-21T13:26:43,113 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/3b98a86e7b0445d0861a39abfd60f058 2024-11-21T13:26:43,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/6706d67c25d74e20bb4c8bffc21b0b17 is 50, key is test_row_0/B:col10/1732195601162/Put/seqid=0 2024-11-21T13:26:43,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742467_1643 (size=12151) 2024-11-21T13:26:43,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:43,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195663155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:43,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:43,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:43,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195663269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:43,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:43,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195663329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:43,331 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:43,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-21T13:26:43,420 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:43,420 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:43,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-21T13:26:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T13:26:43,422 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:43,423 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:43,423 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:43,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:43,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195663479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:43,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/6706d67c25d74e20bb4c8bffc21b0b17 2024-11-21T13:26:43,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/ea66a6cf20774629acf747cc0cb54624 is 50, key is test_row_0/C:col10/1732195601162/Put/seqid=0 2024-11-21T13:26:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T13:26:43,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742468_1644 (size=12151) 2024-11-21T13:26:43,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:43,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195663569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:43,572 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:43,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,573 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:43,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T13:26:43,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:43,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:43,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:43,575 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:43,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:43,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:43,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:43,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195663589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:43,593 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_0' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:43,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T13:26:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:43,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:43,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:43,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:43,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T13:26:43,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:43,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195663789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:43,791 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:43,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T13:26:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:43,880 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:43,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:43,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:43,931 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/ea66a6cf20774629acf747cc0cb54624 2024-11-21T13:26:43,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/3b98a86e7b0445d0861a39abfd60f058 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/3b98a86e7b0445d0861a39abfd60f058 2024-11-21T13:26:43,937 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/3b98a86e7b0445d0861a39abfd60f058, entries=200, sequenceid=173, filesize=38.8 K 2024-11-21T13:26:43,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/6706d67c25d74e20bb4c8bffc21b0b17 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/6706d67c25d74e20bb4c8bffc21b0b17 2024-11-21T13:26:43,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/6706d67c25d74e20bb4c8bffc21b0b17, entries=150, sequenceid=173, filesize=11.9 K 2024-11-21T13:26:43,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/ea66a6cf20774629acf747cc0cb54624 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/ea66a6cf20774629acf747cc0cb54624 2024-11-21T13:26:43,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/ea66a6cf20774629acf747cc0cb54624, entries=150, sequenceid=173, filesize=11.9 K 2024-11-21T13:26:43,944 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 15530bc3c4bda91d823425a8a74e3630 in 864ms, sequenceid=173, compaction requested=true 2024-11-21T13:26:43,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:43,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:43,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:43,945 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:43,945 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:43,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:43,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:43,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:43,945 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:43,947 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102324 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:43,947 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/A is initiating minor compaction (all files) 2024-11-21T13:26:43,947 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/A in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:43,947 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8b3f315041974380be04746fb9680227, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8523ceb9006847ee93d560c68dc31d5b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/3b98a86e7b0445d0861a39abfd60f058] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=99.9 K 2024-11-21T13:26:43,947 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:43,947 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8b3f315041974380be04746fb9680227, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8523ceb9006847ee93d560c68dc31d5b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/3b98a86e7b0445d0861a39abfd60f058] 2024-11-21T13:26:43,947 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:43,947 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/B is initiating minor compaction (all files) 2024-11-21T13:26:43,947 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/B in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:43,948 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/8816cac0afa0476fa4cd5bc1e45cd73c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/04933d7b7f874088b93c76771e8e675e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/6706d67c25d74e20bb4c8bffc21b0b17] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=35.9 K 2024-11-21T13:26:43,948 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b3f315041974380be04746fb9680227, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732195596973 2024-11-21T13:26:43,948 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 8816cac0afa0476fa4cd5bc1e45cd73c, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732195596973 2024-11-21T13:26:43,948 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8523ceb9006847ee93d560c68dc31d5b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732195599400 2024-11-21T13:26:43,948 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 04933d7b7f874088b93c76771e8e675e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732195599400 2024-11-21T13:26:43,948 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b98a86e7b0445d0861a39abfd60f058, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732195601162 2024-11-21T13:26:43,949 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 6706d67c25d74e20bb4c8bffc21b0b17, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732195601162 2024-11-21T13:26:43,955 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#B#compaction#539 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:43,956 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/11e227af68fc4db68f645522ae43e20c is 50, key is test_row_0/B:col10/1732195601162/Put/seqid=0 2024-11-21T13:26:43,961 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:43,965 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121ac596f151d7340029f836420c6d83096_15530bc3c4bda91d823425a8a74e3630 store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:43,967 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121ac596f151d7340029f836420c6d83096_15530bc3c4bda91d823425a8a74e3630, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:43,967 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121ac596f151d7340029f836420c6d83096_15530bc3c4bda91d823425a8a74e3630 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:43,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742469_1645 (size=12561) 2024-11-21T13:26:43,979 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/11e227af68fc4db68f645522ae43e20c as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/11e227af68fc4db68f645522ae43e20c 2024-11-21T13:26:43,985 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/B of 15530bc3c4bda91d823425a8a74e3630 into 11e227af68fc4db68f645522ae43e20c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:43,985 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:43,985 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/B, priority=13, startTime=1732195603945; duration=0sec 2024-11-21T13:26:43,985 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:43,986 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:B 2024-11-21T13:26:43,986 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:43,986 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:43,986 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/C is initiating minor compaction (all files) 2024-11-21T13:26:43,987 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/C in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:43,987 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/792cd9f9289e40c2b70f8b8fb6efce8a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/32b7bbf2d9a7426ab025f441bdb94ac9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/ea66a6cf20774629acf747cc0cb54624] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=35.9 K 2024-11-21T13:26:43,987 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 792cd9f9289e40c2b70f8b8fb6efce8a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1732195596973 2024-11-21T13:26:43,987 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 32b7bbf2d9a7426ab025f441bdb94ac9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1732195599400 2024-11-21T13:26:43,987 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting ea66a6cf20774629acf747cc0cb54624, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732195601162 2024-11-21T13:26:44,003 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#C#compaction#541 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:44,003 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/a17a798e944a44dbadc61b95fa4c36d6 is 50, key is test_row_0/C:col10/1732195601162/Put/seqid=0 2024-11-21T13:26:44,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742470_1646 (size=4469) 2024-11-21T13:26:44,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-21T13:26:44,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:44,033 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-21T13:26:44,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a7784db4fad4444e817c8ebaf02708f0_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195603153/Put/seqid=0 2024-11-21T13:26:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T13:26:44,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742471_1647 (size=12561) 2024-11-21T13:26:44,065 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/a17a798e944a44dbadc61b95fa4c36d6 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/a17a798e944a44dbadc61b95fa4c36d6 2024-11-21T13:26:44,070 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/C of 15530bc3c4bda91d823425a8a74e3630 into a17a798e944a44dbadc61b95fa4c36d6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:44,070 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:44,070 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/C, priority=13, startTime=1732195603945; duration=0sec 2024-11-21T13:26:44,070 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:44,070 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:C 2024-11-21T13:26:44,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742472_1648 (size=12304) 2024-11-21T13:26:44,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:44,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:44,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:44,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195664340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:44,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:44,406 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#A#compaction#540 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:44,406 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/8ba1ecb0d9824831863695b7042f4ab5 is 175, key is test_row_0/A:col10/1732195601162/Put/seqid=0 2024-11-21T13:26:44,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742473_1649 (size=31515) 2024-11-21T13:26:44,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195664449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:44,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:44,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:44,488 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121a7784db4fad4444e817c8ebaf02708f0_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a7784db4fad4444e817c8ebaf02708f0_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:44,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/fb38e314848948dcbc13877673643a84, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:44,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/fb38e314848948dcbc13877673643a84 is 175, key is test_row_0/A:col10/1732195603153/Put/seqid=0 2024-11-21T13:26:44,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742474_1650 (size=31105) 2024-11-21T13:26:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T13:26:44,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:44,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195664659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:44,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:44,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:44,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:44,816 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/8ba1ecb0d9824831863695b7042f4ab5 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8ba1ecb0d9824831863695b7042f4ab5 2024-11-21T13:26:44,820 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/A of 15530bc3c4bda91d823425a8a74e3630 into 8ba1ecb0d9824831863695b7042f4ab5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:44,820 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:44,820 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/A, priority=13, startTime=1732195603945; duration=0sec 2024-11-21T13:26:44,820 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:44,820 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:A 2024-11-21T13:26:44,893 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/fb38e314848948dcbc13877673643a84 2024-11-21T13:26:44,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/18b55dabadb5473cba0cead328ea4b99 is 50, key is test_row_0/B:col10/1732195603153/Put/seqid=0 2024-11-21T13:26:44,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742475_1651 (size=12151) 2024-11-21T13:26:44,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:44,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195664969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:44,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:45,312 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/18b55dabadb5473cba0cead328ea4b99 2024-11-21T13:26:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/bbc6e08594a24b218a12a6e30fab7bd6 is 50, key is test_row_0/C:col10/1732195603153/Put/seqid=0 2024-11-21T13:26:45,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742476_1652 (size=12151) 2024-11-21T13:26:45,321 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/bbc6e08594a24b218a12a6e30fab7bd6 2024-11-21T13:26:45,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/fb38e314848948dcbc13877673643a84 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/fb38e314848948dcbc13877673643a84 2024-11-21T13:26:45,327 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/fb38e314848948dcbc13877673643a84, entries=150, sequenceid=195, filesize=30.4 K 2024-11-21T13:26:45,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/18b55dabadb5473cba0cead328ea4b99 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/18b55dabadb5473cba0cead328ea4b99 2024-11-21T13:26:45,330 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/18b55dabadb5473cba0cead328ea4b99, entries=150, sequenceid=195, filesize=11.9 K 2024-11-21T13:26:45,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/bbc6e08594a24b218a12a6e30fab7bd6 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/bbc6e08594a24b218a12a6e30fab7bd6 2024-11-21T13:26:45,334 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/bbc6e08594a24b218a12a6e30fab7bd6, entries=150, sequenceid=195, filesize=11.9 K 2024-11-21T13:26:45,334 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 15530bc3c4bda91d823425a8a74e3630 in 1301ms, sequenceid=195, compaction requested=false 2024-11-21T13:26:45,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:45,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:45,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-21T13:26:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-21T13:26:45,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-21T13:26:45,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9130 sec 2024-11-21T13:26:45,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.9180 sec 2024-11-21T13:26:45,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:45,352 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:45,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-21T13:26:45,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d7ad589f136447e09bdf72af2458baed_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195604304/Put/seqid=0 2024-11-21T13:26:45,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742477_1653 (size=14794) 2024-11-21T13:26:45,363 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:45,366 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121d7ad589f136447e09bdf72af2458baed_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d7ad589f136447e09bdf72af2458baed_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:45,367 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/8cf6fe8b9fb94bc2b0c409f89c2e6c3a, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:45,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/8cf6fe8b9fb94bc2b0c409f89c2e6c3a is 175, key is test_row_0/A:col10/1732195604304/Put/seqid=0 2024-11-21T13:26:45,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742478_1654 (size=39749) 2024-11-21T13:26:45,424 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:45,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195665421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:45,425 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:45,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:45,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195665479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:45,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:45,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:45,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195665529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:45,531 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-21T13:26:45,569 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:45,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-21T13:26:45,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T13:26:45,571 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:45,572 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:45,572 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:45,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T13:26:45,724 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T13:26:45,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:45,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:45,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:45,724 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:45,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:45,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:45,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195665739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:45,790 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/8cf6fe8b9fb94bc2b0c409f89c2e6c3a 2024-11-21T13:26:45,795 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/d544fdeaf9294286ae5654819e3ddcd6 is 50, key is test_row_0/B:col10/1732195604304/Put/seqid=0 2024-11-21T13:26:45,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742479_1655 (size=12151) 2024-11-21T13:26:45,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T13:26:45,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:45,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:45,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:45,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:45,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:45,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:45,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T13:26:46,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T13:26:46,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:46,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:46,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:46,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:46,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:46,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:46,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:46,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195666049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:46,052 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:46,181 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T13:26:46,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:46,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:46,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:46,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:46,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:46,198 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/d544fdeaf9294286ae5654819e3ddcd6 2024-11-21T13:26:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T13:26:46,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/5d578d47b37a4eb88bc8c817407dff45 is 50, key is test_row_0/C:col10/1732195604304/Put/seqid=0 2024-11-21T13:26:46,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742480_1656 (size=12151) 2024-11-21T13:26:46,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/5d578d47b37a4eb88bc8c817407dff45 2024-11-21T13:26:46,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/8cf6fe8b9fb94bc2b0c409f89c2e6c3a as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8cf6fe8b9fb94bc2b0c409f89c2e6c3a 2024-11-21T13:26:46,236 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8cf6fe8b9fb94bc2b0c409f89c2e6c3a, entries=200, sequenceid=213, filesize=38.8 K 2024-11-21T13:26:46,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/d544fdeaf9294286ae5654819e3ddcd6 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/d544fdeaf9294286ae5654819e3ddcd6 2024-11-21T13:26:46,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/d544fdeaf9294286ae5654819e3ddcd6, entries=150, sequenceid=213, filesize=11.9 K 2024-11-21T13:26:46,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/5d578d47b37a4eb88bc8c817407dff45 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/5d578d47b37a4eb88bc8c817407dff45 2024-11-21T13:26:46,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/5d578d47b37a4eb88bc8c817407dff45, entries=150, sequenceid=213, filesize=11.9 K 2024-11-21T13:26:46,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 15530bc3c4bda91d823425a8a74e3630 in 891ms, sequenceid=213, compaction requested=true 2024-11-21T13:26:46,243 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:46,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:46,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:46,244 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:46,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:46,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:46,244 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:46,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:46,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:46,244 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:46,244 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/B is initiating minor compaction (all files) 2024-11-21T13:26:46,244 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/B in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:46,245 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/11e227af68fc4db68f645522ae43e20c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/18b55dabadb5473cba0cead328ea4b99, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/d544fdeaf9294286ae5654819e3ddcd6] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=36.0 K 2024-11-21T13:26:46,245 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:46,245 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/A is initiating minor compaction (all files) 2024-11-21T13:26:46,245 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/A in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:46,245 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8ba1ecb0d9824831863695b7042f4ab5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/fb38e314848948dcbc13877673643a84, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8cf6fe8b9fb94bc2b0c409f89c2e6c3a] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=100.0 K 2024-11-21T13:26:46,245 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:46,246 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8ba1ecb0d9824831863695b7042f4ab5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/fb38e314848948dcbc13877673643a84, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8cf6fe8b9fb94bc2b0c409f89c2e6c3a] 2024-11-21T13:26:46,246 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 11e227af68fc4db68f645522ae43e20c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732195601162 2024-11-21T13:26:46,246 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 18b55dabadb5473cba0cead328ea4b99, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732195603141 2024-11-21T13:26:46,246 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ba1ecb0d9824831863695b7042f4ab5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732195601162 2024-11-21T13:26:46,246 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting d544fdeaf9294286ae5654819e3ddcd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732195604304 2024-11-21T13:26:46,247 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb38e314848948dcbc13877673643a84, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732195603141 2024-11-21T13:26:46,247 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cf6fe8b9fb94bc2b0c409f89c2e6c3a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732195604304 2024-11-21T13:26:46,252 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:46,253 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#B#compaction#548 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:46,254 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/0121d5368c3b4ed7b63157366f76f2f0 is 50, key is test_row_0/B:col10/1732195604304/Put/seqid=0 2024-11-21T13:26:46,254 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411211f8b5f3d061549fab1553999e9d5dd01_15530bc3c4bda91d823425a8a74e3630 store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:46,255 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411211f8b5f3d061549fab1553999e9d5dd01_15530bc3c4bda91d823425a8a74e3630, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:46,255 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411211f8b5f3d061549fab1553999e9d5dd01_15530bc3c4bda91d823425a8a74e3630 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:46,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742481_1657 (size=12663) 2024-11-21T13:26:46,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742482_1658 (size=4469) 2024-11-21T13:26:46,278 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#A#compaction#549 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:46,278 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/5b1f1b6be2c24465ac6da93036b35e3d is 175, key is test_row_0/A:col10/1732195604304/Put/seqid=0 2024-11-21T13:26:46,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742483_1659 (size=31617) 2024-11-21T13:26:46,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-21T13:26:46,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:46,335 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T13:26:46,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215ace7d52fe9e45ba91a1e1fb5aa24109_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195605386/Put/seqid=0 2024-11-21T13:26:46,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742484_1660 (size=12304) 2024-11-21T13:26:46,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:46,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:46,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:46,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195666546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:46,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:46,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:46,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195666559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:46,561 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:46,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195666659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:46,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:46,671 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/0121d5368c3b4ed7b63157366f76f2f0 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/0121d5368c3b4ed7b63157366f76f2f0 2024-11-21T13:26:46,675 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/B of 15530bc3c4bda91d823425a8a74e3630 into 0121d5368c3b4ed7b63157366f76f2f0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:46,675 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:46,675 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/B, priority=13, startTime=1732195606244; duration=0sec 2024-11-21T13:26:46,675 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:46,675 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:B 2024-11-21T13:26:46,675 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:46,676 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:46,676 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/C is initiating minor compaction (all files) 2024-11-21T13:26:46,676 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/C in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:46,676 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/a17a798e944a44dbadc61b95fa4c36d6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/bbc6e08594a24b218a12a6e30fab7bd6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/5d578d47b37a4eb88bc8c817407dff45] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=36.0 K 2024-11-21T13:26:46,676 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting a17a798e944a44dbadc61b95fa4c36d6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732195601162 2024-11-21T13:26:46,677 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting bbc6e08594a24b218a12a6e30fab7bd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732195603141 2024-11-21T13:26:46,677 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d578d47b37a4eb88bc8c817407dff45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732195604304 2024-11-21T13:26:46,686 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#C#compaction#551 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:46,687 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/2807f13e8437453199a845bdee9aeb20 is 50, key is test_row_0/C:col10/1732195604304/Put/seqid=0 2024-11-21T13:26:46,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742485_1661 (size=12663) 2024-11-21T13:26:46,690 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/5b1f1b6be2c24465ac6da93036b35e3d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/5b1f1b6be2c24465ac6da93036b35e3d 2024-11-21T13:26:46,693 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/A of 15530bc3c4bda91d823425a8a74e3630 into 5b1f1b6be2c24465ac6da93036b35e3d(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:46,693 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:46,693 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/A, priority=13, startTime=1732195606244; duration=0sec 2024-11-21T13:26:46,693 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:46,693 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:A 2024-11-21T13:26:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T13:26:46,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:46,762 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411215ace7d52fe9e45ba91a1e1fb5aa24109_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215ace7d52fe9e45ba91a1e1fb5aa24109_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:46,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/b564e8eddd06417fb11101d7eb66be68, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:46,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/b564e8eddd06417fb11101d7eb66be68 is 175, key is test_row_0/A:col10/1732195605386/Put/seqid=0 2024-11-21T13:26:46,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742486_1662 (size=31105) 2024-11-21T13:26:46,766 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/b564e8eddd06417fb11101d7eb66be68 2024-11-21T13:26:46,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/81d65a402da649869317eb2fcdf39991 is 50, key is test_row_0/B:col10/1732195605386/Put/seqid=0 2024-11-21T13:26:46,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742487_1663 (size=12151) 2024-11-21T13:26:46,781 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/81d65a402da649869317eb2fcdf39991 2024-11-21T13:26:46,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/9ef18e227f7b41c9926cd36b8c248422 is 50, key is test_row_0/C:col10/1732195605386/Put/seqid=0 2024-11-21T13:26:46,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742488_1664 (size=12151) 2024-11-21T13:26:46,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:46,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195666869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:46,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:46,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:46,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:47,094 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/2807f13e8437453199a845bdee9aeb20 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/2807f13e8437453199a845bdee9aeb20 2024-11-21T13:26:47,097 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/C of 15530bc3c4bda91d823425a8a74e3630 into 2807f13e8437453199a845bdee9aeb20(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:47,097 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:47,097 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/C, priority=13, startTime=1732195606244; duration=0sec 2024-11-21T13:26:47,097 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:47,097 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:C 2024-11-21T13:26:47,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:47,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195667179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:47,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:47,194 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/9ef18e227f7b41c9926cd36b8c248422 2024-11-21T13:26:47,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/b564e8eddd06417fb11101d7eb66be68 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/b564e8eddd06417fb11101d7eb66be68 2024-11-21T13:26:47,199 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/b564e8eddd06417fb11101d7eb66be68, entries=150, sequenceid=233, filesize=30.4 K 2024-11-21T13:26:47,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/81d65a402da649869317eb2fcdf39991 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/81d65a402da649869317eb2fcdf39991 2024-11-21T13:26:47,202 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/81d65a402da649869317eb2fcdf39991, entries=150, sequenceid=233, filesize=11.9 K 2024-11-21T13:26:47,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/9ef18e227f7b41c9926cd36b8c248422 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/9ef18e227f7b41c9926cd36b8c248422 2024-11-21T13:26:47,206 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/9ef18e227f7b41c9926cd36b8c248422, entries=150, sequenceid=233, filesize=11.9 K 2024-11-21T13:26:47,206 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 15530bc3c4bda91d823425a8a74e3630 in 872ms, sequenceid=233, compaction requested=false 2024-11-21T13:26:47,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:47,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:47,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-21T13:26:47,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-21T13:26:47,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-21T13:26:47,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6360 sec 2024-11-21T13:26:47,210 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.6400 sec 2024-11-21T13:26:47,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:47,571 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:47,571 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T13:26:47,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f23d0a534312452e8dbec6ebb3b93ad5_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195606530/Put/seqid=0 2024-11-21T13:26:47,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742489_1665 (size=14794) 2024-11-21T13:26:47,596 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:47,602 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121f23d0a534312452e8dbec6ebb3b93ad5_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f23d0a534312452e8dbec6ebb3b93ad5_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:47,602 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/d855aaadfc76478da63d3b27a24bf406, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:47,603 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/d855aaadfc76478da63d3b27a24bf406 is 175, key is test_row_0/A:col10/1732195606530/Put/seqid=0 2024-11-21T13:26:47,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742490_1666 (size=39749) 2024-11-21T13:26:47,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:47,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195667610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:47,614 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/d855aaadfc76478da63d3b27a24bf406 2024-11-21T13:26:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195667613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:47,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:47,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195667614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:47,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/a3c7a7139e86410292d2cf9f1e7ee8be is 50, key is test_row_0/B:col10/1732195606530/Put/seqid=0 2024-11-21T13:26:47,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742491_1667 (size=12151) 2024-11-21T13:26:47,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195667689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:47,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:47,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195667719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,722 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:47,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-21T13:26:47,729 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:47,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-21T13:26:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-21T13:26:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T13:26:47,731 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-21T13:26:47,732 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-21T13:26:47,732 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-21T13:26:47,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195667729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:47,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:47,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195667729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:47,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T13:26:47,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:47,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:47,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:47,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:47,883 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:47,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:47,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:47,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195667929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,931 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:47,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:47,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195667939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:47,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195667939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:47,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:48,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/a3c7a7139e86410292d2cf9f1e7ee8be 2024-11-21T13:26:48,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/464bd760d53943e28846fee5be04b881 is 50, key is test_row_0/C:col10/1732195606530/Put/seqid=0 2024-11-21T13:26:48,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742492_1668 (size=12151) 2024-11-21T13:26:48,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/464bd760d53943e28846fee5be04b881 2024-11-21T13:26:48,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:48,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:48,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/d855aaadfc76478da63d3b27a24bf406 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d855aaadfc76478da63d3b27a24bf406 2024-11-21T13:26:48,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d855aaadfc76478da63d3b27a24bf406, entries=200, sequenceid=253, filesize=38.8 K 2024-11-21T13:26:48,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/a3c7a7139e86410292d2cf9f1e7ee8be as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a3c7a7139e86410292d2cf9f1e7ee8be 2024-11-21T13:26:48,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a3c7a7139e86410292d2cf9f1e7ee8be, entries=150, sequenceid=253, filesize=11.9 K 2024-11-21T13:26:48,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/464bd760d53943e28846fee5be04b881 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/464bd760d53943e28846fee5be04b881 2024-11-21T13:26:48,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T13:26:48,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/464bd760d53943e28846fee5be04b881, entries=150, sequenceid=253, filesize=11.9 K 2024-11-21T13:26:48,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 15530bc3c4bda91d823425a8a74e3630 in 479ms, sequenceid=253, compaction requested=true 2024-11-21T13:26:48,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:48,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:48,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:48,051 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:48,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:48,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:48,051 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:48,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:48,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:48,051 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:48,051 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/A is initiating minor compaction (all files) 2024-11-21T13:26:48,051 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:48,051 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/B is initiating minor compaction (all files) 2024-11-21T13:26:48,051 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/A in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,051 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/B in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,051 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/5b1f1b6be2c24465ac6da93036b35e3d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/b564e8eddd06417fb11101d7eb66be68, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d855aaadfc76478da63d3b27a24bf406] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=100.1 K 2024-11-21T13:26:48,051 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/0121d5368c3b4ed7b63157366f76f2f0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/81d65a402da649869317eb2fcdf39991, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a3c7a7139e86410292d2cf9f1e7ee8be] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=36.1 K 2024-11-21T13:26:48,051 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,051 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/5b1f1b6be2c24465ac6da93036b35e3d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/b564e8eddd06417fb11101d7eb66be68, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d855aaadfc76478da63d3b27a24bf406] 2024-11-21T13:26:48,052 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b1f1b6be2c24465ac6da93036b35e3d, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732195604304 2024-11-21T13:26:48,052 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 0121d5368c3b4ed7b63157366f76f2f0, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732195604304 2024-11-21T13:26:48,052 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 81d65a402da649869317eb2fcdf39991, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732195605386 2024-11-21T13:26:48,052 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting b564e8eddd06417fb11101d7eb66be68, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732195605386 2024-11-21T13:26:48,052 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting a3c7a7139e86410292d2cf9f1e7ee8be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732195606530 2024-11-21T13:26:48,052 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting d855aaadfc76478da63d3b27a24bf406, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732195606530 2024-11-21T13:26:48,056 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:48,057 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#B#compaction#558 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:48,057 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241121e1f15656fc7c416dae5b0ea2bf29868f_15530bc3c4bda91d823425a8a74e3630 store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:48,057 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/8f2d647d25f545799cb4297695ae32b8 is 50, key is test_row_0/B:col10/1732195606530/Put/seqid=0 2024-11-21T13:26:48,059 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241121e1f15656fc7c416dae5b0ea2bf29868f_15530bc3c4bda91d823425a8a74e3630, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:48,059 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121e1f15656fc7c416dae5b0ea2bf29868f_15530bc3c4bda91d823425a8a74e3630 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:48,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742493_1669 (size=12765) 2024-11-21T13:26:48,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742494_1670 (size=4469) 2024-11-21T13:26:48,075 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#A#compaction#557 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:48,076 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/7565af8d82314aa8a6350d604003d515 is 175, key is test_row_0/A:col10/1732195606530/Put/seqid=0 2024-11-21T13:26:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:48,082 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:48,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-21T13:26:48,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121234c5d0faec34cb39ca2a30dde1af110_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195607606/Put/seqid=0 2024-11-21T13:26:48,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742496_1672 (size=12454) 2024-11-21T13:26:48,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742495_1671 (size=31719) 2024-11-21T13:26:48,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58350 deadline: 1732195668134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:48,188 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:48,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:48,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:48,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195668239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,243 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:48,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:48,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195668249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,253 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195668249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:48,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58350 deadline: 1732195668249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:48,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:48,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:48,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,341 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T13:26:48,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58350 deadline: 1732195668459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:48,473 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/8f2d647d25f545799cb4297695ae32b8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/8f2d647d25f545799cb4297695ae32b8 2024-11-21T13:26:48,477 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/B of 15530bc3c4bda91d823425a8a74e3630 into 8f2d647d25f545799cb4297695ae32b8(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:48,477 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:48,477 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/B, priority=13, startTime=1732195608051; duration=0sec 2024-11-21T13:26:48,477 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:48,477 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:B 2024-11-21T13:26:48,477 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:48,478 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:48,478 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/C is initiating minor compaction (all files) 2024-11-21T13:26:48,478 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/C in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,478 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/2807f13e8437453199a845bdee9aeb20, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/9ef18e227f7b41c9926cd36b8c248422, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/464bd760d53943e28846fee5be04b881] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=36.1 K 2024-11-21T13:26:48,479 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 2807f13e8437453199a845bdee9aeb20, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1732195604304 2024-11-21T13:26:48,479 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 9ef18e227f7b41c9926cd36b8c248422, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1732195605386 2024-11-21T13:26:48,479 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 464bd760d53943e28846fee5be04b881, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732195606530 2024-11-21T13:26:48,485 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#C#compaction#560 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:48,485 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/f3c99ba49a264ad0897d871d75b9ba65 is 50, key is test_row_0/C:col10/1732195606530/Put/seqid=0 2024-11-21T13:26:48,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742497_1673 (size=12765) 2024-11-21T13:26:48,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:48,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:48,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,499 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:48,502 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241121234c5d0faec34cb39ca2a30dde1af110_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121234c5d0faec34cb39ca2a30dde1af110_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:48,503 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/a62d524a8a8a404caec4a7b98054181d, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:48,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/a62d524a8a8a404caec4a7b98054181d is 175, key is test_row_0/A:col10/1732195607606/Put/seqid=0 2024-11-21T13:26:48,504 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/7565af8d82314aa8a6350d604003d515 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/7565af8d82314aa8a6350d604003d515 2024-11-21T13:26:48,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742498_1674 (size=31255) 2024-11-21T13:26:48,508 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/A of 15530bc3c4bda91d823425a8a74e3630 into 7565af8d82314aa8a6350d604003d515(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:48,508 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:48,508 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/A, priority=13, startTime=1732195608051; duration=0sec 2024-11-21T13:26:48,508 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:48,508 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:A 2024-11-21T13:26:48,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:48,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:48,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:48,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58352 deadline: 1732195668709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:48,710 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 4fe6efa1e611,33573,1732195430851 for 'test_row_2' in 15530bc3c4bda91d823425a8a74e3630 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-21T13:26:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,710 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:48,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:48,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58416 deadline: 1732195668749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:48,750 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:48,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:48,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58394 deadline: 1732195668759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:48,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:48,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58364 deadline: 1732195668769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:48,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:48,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58350 deadline: 1732195668770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:48,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:48,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:48,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:48,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T13:26:48,891 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/f3c99ba49a264ad0897d871d75b9ba65 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f3c99ba49a264ad0897d871d75b9ba65 2024-11-21T13:26:48,895 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/C of 15530bc3c4bda91d823425a8a74e3630 into f3c99ba49a264ad0897d871d75b9ba65(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:48,895 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:48,895 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/C, priority=13, startTime=1732195608051; duration=0sec 2024-11-21T13:26:48,895 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:48,895 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:C 2024-11-21T13:26:48,908 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/a62d524a8a8a404caec4a7b98054181d 2024-11-21T13:26:48,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/1968132193204bf8bf622fa9c3c36dcd is 50, key is test_row_0/B:col10/1732195607606/Put/seqid=0 2024-11-21T13:26:48,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742499_1675 (size=12301) 2024-11-21T13:26:48,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:48,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:48,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:49,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,231 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-21T13:26:49,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:49,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:49,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:49,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:49,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T13:26:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33573 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:58350 deadline: 1732195669279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 , the old value is region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-21T13:26:49,281 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., hostname=4fe6efa1e611,33573,1732195430851, seqNum=5 because the exception is null or not the one we care about 2024-11-21T13:26:49,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/1968132193204bf8bf622fa9c3c36dcd 2024-11-21T13:26:49,333 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/801c2c4ce38c43dca8a24260816cc3e4 is 50, key is test_row_0/C:col10/1732195607606/Put/seqid=0 2024-11-21T13:26:49,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742500_1676 (size=12301) 2024-11-21T13:26:49,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:49,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:49,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:49,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:49,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:49,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:49,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:49,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:49,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:49,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:49,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:49,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:49,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/801c2c4ce38c43dca8a24260816cc3e4 2024-11-21T13:26:49,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/a62d524a8a8a404caec4a7b98054181d as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/a62d524a8a8a404caec4a7b98054181d 2024-11-21T13:26:49,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/a62d524a8a8a404caec4a7b98054181d, entries=150, sequenceid=273, filesize=30.5 K 2024-11-21T13:26:49,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/1968132193204bf8bf622fa9c3c36dcd as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/1968132193204bf8bf622fa9c3c36dcd 2024-11-21T13:26:49,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/1968132193204bf8bf622fa9c3c36dcd, entries=150, sequenceid=273, filesize=12.0 K 2024-11-21T13:26:49,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/801c2c4ce38c43dca8a24260816cc3e4 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/801c2c4ce38c43dca8a24260816cc3e4 2024-11-21T13:26:49,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/801c2c4ce38c43dca8a24260816cc3e4, entries=150, sequenceid=273, filesize=12.0 K 2024-11-21T13:26:49,762 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 15530bc3c4bda91d823425a8a74e3630 in 1680ms, sequenceid=273, compaction requested=false 2024-11-21T13:26:49,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33573 {}] regionserver.HRegion(8855): Flush requested on 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:49,763 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-21T13:26:49,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-21T13:26:49,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112102063bf65dc24393b5c39853466e9af2_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195609762/Put/seqid=0 2024-11-21T13:26:49,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742501_1677 (size=14994) 2024-11-21T13:26:49,793 INFO [Thread-2632 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2632. 2024-11-21T13:26:49,793 DEBUG [Thread-2632 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:49,793 DEBUG [Thread-2632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,793 DEBUG [Thread-2632 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:49,794 INFO [Thread-2645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2645. 2024-11-21T13:26:49,794 DEBUG [Thread-2645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:49,794 DEBUG [Thread-2645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,794 DEBUG [Thread-2645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:49,794 INFO [Thread-2636 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2636. 2024-11-21T13:26:49,794 DEBUG [Thread-2636 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:49,794 DEBUG [Thread-2636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,794 DEBUG [Thread-2636 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:49,795 INFO [Thread-2643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2643. 2024-11-21T13:26:49,795 DEBUG [Thread-2643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:49,795 DEBUG [Thread-2643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,795 DEBUG [Thread-2643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:49,797 INFO [Thread-2647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2647. 2024-11-21T13:26:49,797 DEBUG [Thread-2647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:49,797 DEBUG [Thread-2647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,797 DEBUG [Thread-2647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:49,797 INFO [Thread-2640 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2640. 2024-11-21T13:26:49,797 DEBUG [Thread-2640 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:49,797 DEBUG [Thread-2640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,797 DEBUG [Thread-2640 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,797 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:49,798 INFO [Thread-2649 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2649. 2024-11-21T13:26:49,798 DEBUG [Thread-2649 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:49,798 DEBUG [Thread-2649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,798 INFO [Thread-2651 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2651. 2024-11-21T13:26:49,798 DEBUG [Thread-2649 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,798 DEBUG [Thread-2651 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:49,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:49,798 DEBUG [Thread-2651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,798 DEBUG [Thread-2651 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:49,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:49,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:49,866 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T13:26:50,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:50,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:50,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,019 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:50,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:50,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,183 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:50,185 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112102063bf65dc24393b5c39853466e9af2_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112102063bf65dc24393b5c39853466e9af2_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:50,186 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/4518abc7d0ee46b5928a63d18a8bfd4e, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:50,186 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/4518abc7d0ee46b5928a63d18a8bfd4e is 175, key is test_row_0/A:col10/1732195609762/Put/seqid=0 2024-11-21T13:26:50,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742502_1678 (size=39949) 2024-11-21T13:26:50,291 INFO [Thread-2634 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2634. 2024-11-21T13:26:50,291 DEBUG [Thread-2634 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:50,291 DEBUG [Thread-2634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:50,291 DEBUG [Thread-2634 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:50,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:50,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:50,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:50,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,476 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:50,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:50,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,476 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,591 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/4518abc7d0ee46b5928a63d18a8bfd4e 2024-11-21T13:26:50,595 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/d695d6f37f00429094b11f3e49365971 is 50, key is test_row_0/B:col10/1732195609762/Put/seqid=0 2024-11-21T13:26:50,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742503_1679 (size=12301) 2024-11-21T13:26:50,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:50,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,721 INFO [Thread-2638 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2638. 2024-11-21T13:26:50,721 DEBUG [Thread-2638 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-21T13:26:50,721 DEBUG [Thread-2638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:50,721 DEBUG [Thread-2638 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:50,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:50,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:50,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:50,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,781 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,934 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:50,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:50,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:50,934 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:50,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/d695d6f37f00429094b11f3e49365971 2024-11-21T13:26:51,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/e3acf8f91c8047c5826b3ec0ce47a732 is 50, key is test_row_0/C:col10/1732195609762/Put/seqid=0 2024-11-21T13:26:51,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742504_1680 (size=12301) 2024-11-21T13:26:51,086 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:51,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:51,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:51,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:51,087 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:51,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:51,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:51,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:51,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:51,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:51,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:51,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:51,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:51,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:51,392 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:51,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:51,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. as already flushing 2024-11-21T13:26:51,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:51,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:51,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:51,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T13:26:51,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/e3acf8f91c8047c5826b3ec0ce47a732 2024-11-21T13:26:51,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/4518abc7d0ee46b5928a63d18a8bfd4e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/4518abc7d0ee46b5928a63d18a8bfd4e 2024-11-21T13:26:51,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/4518abc7d0ee46b5928a63d18a8bfd4e, entries=200, sequenceid=293, filesize=39.0 K 2024-11-21T13:26:51,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/d695d6f37f00429094b11f3e49365971 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/d695d6f37f00429094b11f3e49365971 2024-11-21T13:26:51,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/d695d6f37f00429094b11f3e49365971, entries=150, sequenceid=293, filesize=12.0 K 2024-11-21T13:26:51,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/e3acf8f91c8047c5826b3ec0ce47a732 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/e3acf8f91c8047c5826b3ec0ce47a732 2024-11-21T13:26:51,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/e3acf8f91c8047c5826b3ec0ce47a732, entries=150, sequenceid=293, filesize=12.0 K 2024-11-21T13:26:51,419 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 15530bc3c4bda91d823425a8a74e3630 in 1656ms, sequenceid=293, compaction requested=true 2024-11-21T13:26:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:A, priority=-2147483648, current under compaction store size is 1 2024-11-21T13:26:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:51,419 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:B, priority=-2147483648, current under compaction store size is 2 2024-11-21T13:26:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:51,419 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 15530bc3c4bda91d823425a8a74e3630:C, priority=-2147483648, current under compaction store size is 3 2024-11-21T13:26:51,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:51,419 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102923 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:51,420 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/A is initiating minor compaction (all files) 2024-11-21T13:26:51,420 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:51,420 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/B is initiating minor compaction (all files) 2024-11-21T13:26:51,420 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/A in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:51,420 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/B in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:51,420 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/7565af8d82314aa8a6350d604003d515, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/a62d524a8a8a404caec4a7b98054181d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/4518abc7d0ee46b5928a63d18a8bfd4e] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=100.5 K 2024-11-21T13:26:51,420 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/8f2d647d25f545799cb4297695ae32b8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/1968132193204bf8bf622fa9c3c36dcd, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/d695d6f37f00429094b11f3e49365971] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=36.5 K 2024-11-21T13:26:51,420 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:51,420 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. files: [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/7565af8d82314aa8a6350d604003d515, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/a62d524a8a8a404caec4a7b98054181d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/4518abc7d0ee46b5928a63d18a8bfd4e] 2024-11-21T13:26:51,420 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f2d647d25f545799cb4297695ae32b8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732195606530 2024-11-21T13:26:51,420 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7565af8d82314aa8a6350d604003d515, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732195606530 2024-11-21T13:26:51,420 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 1968132193204bf8bf622fa9c3c36dcd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732195607606 2024-11-21T13:26:51,420 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting a62d524a8a8a404caec4a7b98054181d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732195607606 2024-11-21T13:26:51,420 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting d695d6f37f00429094b11f3e49365971, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732195608117 2024-11-21T13:26:51,420 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4518abc7d0ee46b5928a63d18a8bfd4e, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732195608117 2024-11-21T13:26:51,426 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#B#compaction#566 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:51,427 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/bea47412f37740829cae15ec4b259b57 is 50, key is test_row_0/B:col10/1732195609762/Put/seqid=0 2024-11-21T13:26:51,428 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:51,430 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411216781bbacd5e14cab8c003bb06852302e_15530bc3c4bda91d823425a8a74e3630 store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:51,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742505_1681 (size=13017) 2024-11-21T13:26:51,458 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411216781bbacd5e14cab8c003bb06852302e_15530bc3c4bda91d823425a8a74e3630, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:51,458 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411216781bbacd5e14cab8c003bb06852302e_15530bc3c4bda91d823425a8a74e3630 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:51,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742506_1682 (size=4469) 2024-11-21T13:26:51,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33573 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-21T13:26:51,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:51,545 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 15530bc3c4bda91d823425a8a74e3630 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-21T13:26:51,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112140bdf562b9ec4b1ba85bb4e196167c37_15530bc3c4bda91d823425a8a74e3630 is 50, key is test_row_0/A:col10/1732195609787/Put/seqid=0 2024-11-21T13:26:51,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742507_1683 (size=12454) 2024-11-21T13:26:51,834 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/bea47412f37740829cae15ec4b259b57 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/bea47412f37740829cae15ec4b259b57 2024-11-21T13:26:51,838 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/B of 15530bc3c4bda91d823425a8a74e3630 into bea47412f37740829cae15ec4b259b57(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:51,838 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:51,838 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/B, priority=13, startTime=1732195611419; duration=0sec 2024-11-21T13:26:51,838 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-21T13:26:51,838 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:B 2024-11-21T13:26:51,838 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-21T13:26:51,839 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-21T13:26:51,839 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1541): 15530bc3c4bda91d823425a8a74e3630/C is initiating minor compaction (all files) 2024-11-21T13:26:51,839 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 15530bc3c4bda91d823425a8a74e3630/C in TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:51,839 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f3c99ba49a264ad0897d871d75b9ba65, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/801c2c4ce38c43dca8a24260816cc3e4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/e3acf8f91c8047c5826b3ec0ce47a732] into tmpdir=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp, totalSize=36.5 K 2024-11-21T13:26:51,839 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting f3c99ba49a264ad0897d871d75b9ba65, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732195606530 2024-11-21T13:26:51,839 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting 801c2c4ce38c43dca8a24260816cc3e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1732195607606 2024-11-21T13:26:51,840 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] compactions.Compactor(225): Compacting e3acf8f91c8047c5826b3ec0ce47a732, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1732195608117 2024-11-21T13:26:51,845 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#C#compaction#569 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:51,845 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/7773b5756080446b8db0ff43b214dd17 is 50, key is test_row_0/C:col10/1732195609762/Put/seqid=0 2024-11-21T13:26:51,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742508_1684 (size=13017) 2024-11-21T13:26:51,862 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 15530bc3c4bda91d823425a8a74e3630#A#compaction#567 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-21T13:26:51,863 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/d9f88959b32f491fa84c4418e2c8edae is 175, key is test_row_0/A:col10/1732195609762/Put/seqid=0 2024-11-21T13:26:51,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742509_1685 (size=31971) 2024-11-21T13:26:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T13:26:51,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T13:26:51,966 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112140bdf562b9ec4b1ba85bb4e196167c37_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112140bdf562b9ec4b1ba85bb4e196167c37_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/6d563eb5f8014f8fbf572f75737a1a51, store: [table=TestAcidGuarantees family=A region=15530bc3c4bda91d823425a8a74e3630] 2024-11-21T13:26:51,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/6d563eb5f8014f8fbf572f75737a1a51 is 175, key is test_row_0/A:col10/1732195609787/Put/seqid=0 2024-11-21T13:26:51,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742510_1686 (size=31255) 2024-11-21T13:26:52,251 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/7773b5756080446b8db0ff43b214dd17 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/7773b5756080446b8db0ff43b214dd17 2024-11-21T13:26:52,254 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/C of 15530bc3c4bda91d823425a8a74e3630 into 7773b5756080446b8db0ff43b214dd17(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:52,255 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:52,255 INFO [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/C, priority=13, startTime=1732195611419; duration=0sec 2024-11-21T13:26:52,255 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:52,255 DEBUG [RS:0;4fe6efa1e611:33573-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:C 2024-11-21T13:26:52,269 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/d9f88959b32f491fa84c4418e2c8edae as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d9f88959b32f491fa84c4418e2c8edae 2024-11-21T13:26:52,273 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 15530bc3c4bda91d823425a8a74e3630/A of 15530bc3c4bda91d823425a8a74e3630 into d9f88959b32f491fa84c4418e2c8edae(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-21T13:26:52,273 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:52,273 INFO [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630., storeName=15530bc3c4bda91d823425a8a74e3630/A, priority=13, startTime=1732195611419; duration=0sec 2024-11-21T13:26:52,273 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-21T13:26:52,273 DEBUG [RS:0;4fe6efa1e611:33573-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 15530bc3c4bda91d823425a8a74e3630:A 2024-11-21T13:26:52,371 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/6d563eb5f8014f8fbf572f75737a1a51 2024-11-21T13:26:52,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/ffddf790238f44e5b374c749846e825e is 50, key is test_row_0/B:col10/1732195609787/Put/seqid=0 2024-11-21T13:26:52,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742511_1687 (size=12301) 2024-11-21T13:26:52,779 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/ffddf790238f44e5b374c749846e825e 2024-11-21T13:26:52,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/013fd4feddb147128dc5c19134f051e3 is 50, key is test_row_0/C:col10/1732195609787/Put/seqid=0 2024-11-21T13:26:52,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742512_1688 (size=12301) 2024-11-21T13:26:53,188 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/013fd4feddb147128dc5c19134f051e3 2024-11-21T13:26:53,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/A/6d563eb5f8014f8fbf572f75737a1a51 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/6d563eb5f8014f8fbf572f75737a1a51 2024-11-21T13:26:53,194 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/6d563eb5f8014f8fbf572f75737a1a51, entries=150, sequenceid=312, filesize=30.5 K 2024-11-21T13:26:53,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/B/ffddf790238f44e5b374c749846e825e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/ffddf790238f44e5b374c749846e825e 2024-11-21T13:26:53,197 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/ffddf790238f44e5b374c749846e825e, entries=150, sequenceid=312, filesize=12.0 K 2024-11-21T13:26:53,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/.tmp/C/013fd4feddb147128dc5c19134f051e3 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/013fd4feddb147128dc5c19134f051e3 2024-11-21T13:26:53,200 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/013fd4feddb147128dc5c19134f051e3, entries=150, sequenceid=312, filesize=12.0 K 2024-11-21T13:26:53,201 INFO [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=0 B/0 for 15530bc3c4bda91d823425a8a74e3630 in 1656ms, sequenceid=312, compaction requested=false 2024-11-21T13:26:53,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 15530bc3c4bda91d823425a8a74e3630: 2024-11-21T13:26:53,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:53,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/4fe6efa1e611:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-21T13:26:53,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-21T13:26:53,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-21T13:26:53,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 5.4700 sec 2024-11-21T13:26:53,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 5.4740 sec 2024-11-21T13:26:55,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-21T13:26:55,940 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:55,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-21T13:26:55,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-21T13:26:55,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 21 2024-11-21T13:26:55,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-21T13:26:55,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-21T13:26:55,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-21T13:26:55,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-21T13:26:55,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4324 2024-11-21T13:26:55,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4222 2024-11-21T13:26:55,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4092 2024-11-21T13:26:55,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4339 2024-11-21T13:26:55,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4240 2024-11-21T13:26:55,940 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-21T13:26:55,940 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T13:26:55,940 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T13:26:55,940 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:55,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:55,941 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:55,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-21T13:26:55,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T13:26:55,944 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195615944"}]},"ts":"1732195615944"} 2024-11-21T13:26:55,945 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-21T13:26:55,945 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-21T13:26:55,945 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-21T13:26:55,947 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, UNASSIGN}] 2024-11-21T13:26:55,947 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, UNASSIGN 2024-11-21T13:26:55,948 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=15530bc3c4bda91d823425a8a74e3630, regionState=CLOSING, regionLocation=4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:55,949 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, UNASSIGN because future has completed 2024-11-21T13:26:55,949 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-21T13:26:55,949 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851}] 2024-11-21T13:26:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T13:26:56,102 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,102 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-21T13:26:56,102 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing 15530bc3c4bda91d823425a8a74e3630, disabling compactions & flushes 2024-11-21T13:26:56,102 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:56,102 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:56,102 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. after waiting 0 ms 2024-11-21T13:26:56,102 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:56,102 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/1f42cec327044ad8846a4560b62d1743, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c623f805d6da4e3eb02a9e81e867ee93, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/db16f4e7772d4cb29d43ca5985667582, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c00a7bae26534672abff176a6873d995, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/68e1cdec24de4470a5b8cb3e7a56d9cd, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d5deddeb715f45ac93c554cf1b301d5c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d3172bf5ee98432cac99fdb93ce263d5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/bd94f0b39ce3498ab35506689666f76e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8b3f315041974380be04746fb9680227, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/0d04edc744554e589d032576327d5a35, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8523ceb9006847ee93d560c68dc31d5b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/3b98a86e7b0445d0861a39abfd60f058, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8ba1ecb0d9824831863695b7042f4ab5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/fb38e314848948dcbc13877673643a84, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8cf6fe8b9fb94bc2b0c409f89c2e6c3a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/5b1f1b6be2c24465ac6da93036b35e3d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/b564e8eddd06417fb11101d7eb66be68, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d855aaadfc76478da63d3b27a24bf406, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/7565af8d82314aa8a6350d604003d515, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/a62d524a8a8a404caec4a7b98054181d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/4518abc7d0ee46b5928a63d18a8bfd4e] to archive 2024-11-21T13:26:56,103 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:26:56,105 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/1f42cec327044ad8846a4560b62d1743 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/1f42cec327044ad8846a4560b62d1743 2024-11-21T13:26:56,106 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c623f805d6da4e3eb02a9e81e867ee93 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c623f805d6da4e3eb02a9e81e867ee93 2024-11-21T13:26:56,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/db16f4e7772d4cb29d43ca5985667582 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/db16f4e7772d4cb29d43ca5985667582 2024-11-21T13:26:56,107 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c00a7bae26534672abff176a6873d995 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/c00a7bae26534672abff176a6873d995 2024-11-21T13:26:56,108 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/68e1cdec24de4470a5b8cb3e7a56d9cd to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/68e1cdec24de4470a5b8cb3e7a56d9cd 2024-11-21T13:26:56,109 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d5deddeb715f45ac93c554cf1b301d5c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d5deddeb715f45ac93c554cf1b301d5c 2024-11-21T13:26:56,110 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d3172bf5ee98432cac99fdb93ce263d5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d3172bf5ee98432cac99fdb93ce263d5 2024-11-21T13:26:56,111 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/bd94f0b39ce3498ab35506689666f76e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/bd94f0b39ce3498ab35506689666f76e 2024-11-21T13:26:56,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8b3f315041974380be04746fb9680227 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8b3f315041974380be04746fb9680227 2024-11-21T13:26:56,112 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/0d04edc744554e589d032576327d5a35 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/0d04edc744554e589d032576327d5a35 2024-11-21T13:26:56,113 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8523ceb9006847ee93d560c68dc31d5b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8523ceb9006847ee93d560c68dc31d5b 2024-11-21T13:26:56,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/3b98a86e7b0445d0861a39abfd60f058 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/3b98a86e7b0445d0861a39abfd60f058 2024-11-21T13:26:56,114 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8ba1ecb0d9824831863695b7042f4ab5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8ba1ecb0d9824831863695b7042f4ab5 2024-11-21T13:26:56,115 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/fb38e314848948dcbc13877673643a84 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/fb38e314848948dcbc13877673643a84 2024-11-21T13:26:56,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8cf6fe8b9fb94bc2b0c409f89c2e6c3a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/8cf6fe8b9fb94bc2b0c409f89c2e6c3a 2024-11-21T13:26:56,116 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/5b1f1b6be2c24465ac6da93036b35e3d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/5b1f1b6be2c24465ac6da93036b35e3d 2024-11-21T13:26:56,117 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/b564e8eddd06417fb11101d7eb66be68 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/b564e8eddd06417fb11101d7eb66be68 2024-11-21T13:26:56,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d855aaadfc76478da63d3b27a24bf406 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d855aaadfc76478da63d3b27a24bf406 2024-11-21T13:26:56,118 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/7565af8d82314aa8a6350d604003d515 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/7565af8d82314aa8a6350d604003d515 2024-11-21T13:26:56,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/a62d524a8a8a404caec4a7b98054181d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/a62d524a8a8a404caec4a7b98054181d 2024-11-21T13:26:56,119 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/4518abc7d0ee46b5928a63d18a8bfd4e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/4518abc7d0ee46b5928a63d18a8bfd4e 2024-11-21T13:26:56,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a317c87798664ee78300042b1a70c1ca, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/9342136bfdb4474396b9fe3377ac38c7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/c5bdfea323c441ec91f754d8f9148dc8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/f15c29d19e22495d86f5dd1c1d6b77e5, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/1692c4f2420d4b2b817da5cd677eed12, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/29de47f09e52450ca78fe6ab3837dac1, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/44eb52f65350443ebcc5507be30aae22, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/e55ea4cb29d54b1a912fbf40cb449a82, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/8816cac0afa0476fa4cd5bc1e45cd73c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a618ea62b098400b82b107791bfd6caf, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/04933d7b7f874088b93c76771e8e675e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/11e227af68fc4db68f645522ae43e20c, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/6706d67c25d74e20bb4c8bffc21b0b17, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/18b55dabadb5473cba0cead328ea4b99, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/0121d5368c3b4ed7b63157366f76f2f0, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/d544fdeaf9294286ae5654819e3ddcd6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/81d65a402da649869317eb2fcdf39991, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/8f2d647d25f545799cb4297695ae32b8, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a3c7a7139e86410292d2cf9f1e7ee8be, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/1968132193204bf8bf622fa9c3c36dcd, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/d695d6f37f00429094b11f3e49365971] to archive 2024-11-21T13:26:56,121 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:26:56,122 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a317c87798664ee78300042b1a70c1ca to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a317c87798664ee78300042b1a70c1ca 2024-11-21T13:26:56,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/9342136bfdb4474396b9fe3377ac38c7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/9342136bfdb4474396b9fe3377ac38c7 2024-11-21T13:26:56,123 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/c5bdfea323c441ec91f754d8f9148dc8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/c5bdfea323c441ec91f754d8f9148dc8 2024-11-21T13:26:56,124 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/f15c29d19e22495d86f5dd1c1d6b77e5 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/f15c29d19e22495d86f5dd1c1d6b77e5 2024-11-21T13:26:56,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/1692c4f2420d4b2b817da5cd677eed12 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/1692c4f2420d4b2b817da5cd677eed12 2024-11-21T13:26:56,125 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/29de47f09e52450ca78fe6ab3837dac1 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/29de47f09e52450ca78fe6ab3837dac1 2024-11-21T13:26:56,126 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/44eb52f65350443ebcc5507be30aae22 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/44eb52f65350443ebcc5507be30aae22 2024-11-21T13:26:56,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/e55ea4cb29d54b1a912fbf40cb449a82 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/e55ea4cb29d54b1a912fbf40cb449a82 2024-11-21T13:26:56,127 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/8816cac0afa0476fa4cd5bc1e45cd73c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/8816cac0afa0476fa4cd5bc1e45cd73c 2024-11-21T13:26:56,128 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a618ea62b098400b82b107791bfd6caf to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a618ea62b098400b82b107791bfd6caf 2024-11-21T13:26:56,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/04933d7b7f874088b93c76771e8e675e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/04933d7b7f874088b93c76771e8e675e 2024-11-21T13:26:56,129 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/11e227af68fc4db68f645522ae43e20c to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/11e227af68fc4db68f645522ae43e20c 2024-11-21T13:26:56,130 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/6706d67c25d74e20bb4c8bffc21b0b17 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/6706d67c25d74e20bb4c8bffc21b0b17 2024-11-21T13:26:56,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/18b55dabadb5473cba0cead328ea4b99 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/18b55dabadb5473cba0cead328ea4b99 2024-11-21T13:26:56,131 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/0121d5368c3b4ed7b63157366f76f2f0 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/0121d5368c3b4ed7b63157366f76f2f0 2024-11-21T13:26:56,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/d544fdeaf9294286ae5654819e3ddcd6 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/d544fdeaf9294286ae5654819e3ddcd6 2024-11-21T13:26:56,132 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/81d65a402da649869317eb2fcdf39991 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/81d65a402da649869317eb2fcdf39991 2024-11-21T13:26:56,133 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/8f2d647d25f545799cb4297695ae32b8 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/8f2d647d25f545799cb4297695ae32b8 2024-11-21T13:26:56,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a3c7a7139e86410292d2cf9f1e7ee8be to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/a3c7a7139e86410292d2cf9f1e7ee8be 2024-11-21T13:26:56,134 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/1968132193204bf8bf622fa9c3c36dcd to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/1968132193204bf8bf622fa9c3c36dcd 2024-11-21T13:26:56,135 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/d695d6f37f00429094b11f3e49365971 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/d695d6f37f00429094b11f3e49365971 2024-11-21T13:26:56,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f1960ec8999d4c559933343ebe2e236b, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/b05115e213a94f6089341fbeb53038a9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/5cd0d79889444361a4bb52104becd9a7, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/88804529cc514680ba0f400deca7a7aa, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/ffdf4abdc4224bdda017b573f3fc0300, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/2edfc464bf16498ea616ecd7eea1143e, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/9e962c0cc32643e69342c9e22c62e879, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/0648d2bae07b4222871e32ff436d79ff, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/792cd9f9289e40c2b70f8b8fb6efce8a, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f2d97b0a855a4af6af04c3630f77016d, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/32b7bbf2d9a7426ab025f441bdb94ac9, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/a17a798e944a44dbadc61b95fa4c36d6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/ea66a6cf20774629acf747cc0cb54624, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/bbc6e08594a24b218a12a6e30fab7bd6, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/2807f13e8437453199a845bdee9aeb20, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/5d578d47b37a4eb88bc8c817407dff45, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/9ef18e227f7b41c9926cd36b8c248422, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f3c99ba49a264ad0897d871d75b9ba65, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/464bd760d53943e28846fee5be04b881, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/801c2c4ce38c43dca8a24260816cc3e4, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/e3acf8f91c8047c5826b3ec0ce47a732] to archive 2024-11-21T13:26:56,136 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-21T13:26:56,137 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f1960ec8999d4c559933343ebe2e236b to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f1960ec8999d4c559933343ebe2e236b 2024-11-21T13:26:56,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/b05115e213a94f6089341fbeb53038a9 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/b05115e213a94f6089341fbeb53038a9 2024-11-21T13:26:56,138 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/5cd0d79889444361a4bb52104becd9a7 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/5cd0d79889444361a4bb52104becd9a7 2024-11-21T13:26:56,139 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/88804529cc514680ba0f400deca7a7aa to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/88804529cc514680ba0f400deca7a7aa 2024-11-21T13:26:56,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/ffdf4abdc4224bdda017b573f3fc0300 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/ffdf4abdc4224bdda017b573f3fc0300 2024-11-21T13:26:56,140 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/2edfc464bf16498ea616ecd7eea1143e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/2edfc464bf16498ea616ecd7eea1143e 2024-11-21T13:26:56,141 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/9e962c0cc32643e69342c9e22c62e879 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/9e962c0cc32643e69342c9e22c62e879 2024-11-21T13:26:56,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/0648d2bae07b4222871e32ff436d79ff to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/0648d2bae07b4222871e32ff436d79ff 2024-11-21T13:26:56,142 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/792cd9f9289e40c2b70f8b8fb6efce8a to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/792cd9f9289e40c2b70f8b8fb6efce8a 2024-11-21T13:26:56,143 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f2d97b0a855a4af6af04c3630f77016d to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f2d97b0a855a4af6af04c3630f77016d 2024-11-21T13:26:56,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/32b7bbf2d9a7426ab025f441bdb94ac9 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/32b7bbf2d9a7426ab025f441bdb94ac9 2024-11-21T13:26:56,144 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/a17a798e944a44dbadc61b95fa4c36d6 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/a17a798e944a44dbadc61b95fa4c36d6 2024-11-21T13:26:56,145 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/ea66a6cf20774629acf747cc0cb54624 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/ea66a6cf20774629acf747cc0cb54624 2024-11-21T13:26:56,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/bbc6e08594a24b218a12a6e30fab7bd6 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/bbc6e08594a24b218a12a6e30fab7bd6 2024-11-21T13:26:56,146 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/2807f13e8437453199a845bdee9aeb20 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/2807f13e8437453199a845bdee9aeb20 2024-11-21T13:26:56,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/5d578d47b37a4eb88bc8c817407dff45 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/5d578d47b37a4eb88bc8c817407dff45 2024-11-21T13:26:56,147 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/9ef18e227f7b41c9926cd36b8c248422 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/9ef18e227f7b41c9926cd36b8c248422 2024-11-21T13:26:56,148 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f3c99ba49a264ad0897d871d75b9ba65 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/f3c99ba49a264ad0897d871d75b9ba65 2024-11-21T13:26:56,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/464bd760d53943e28846fee5be04b881 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/464bd760d53943e28846fee5be04b881 2024-11-21T13:26:56,149 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/801c2c4ce38c43dca8a24260816cc3e4 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/801c2c4ce38c43dca8a24260816cc3e4 2024-11-21T13:26:56,150 DEBUG [StoreCloser-TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/e3acf8f91c8047c5826b3ec0ce47a732 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/e3acf8f91c8047c5826b3ec0ce47a732 2024-11-21T13:26:56,153 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/recovered.edits/318.seqid, newMaxSeqId=318, maxSeqId=4 2024-11-21T13:26:56,153 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630. 2024-11-21T13:26:56,153 DEBUG [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for 15530bc3c4bda91d823425a8a74e3630: Waiting for close lock at 1732195616102Running coprocessor pre-close hooks at 1732195616102Disabling compacts and flushes for region at 1732195616102Disabling writes for close at 1732195616102Writing region close event to WAL at 1732195616151 (+49 ms)Running coprocessor post-close hooks at 1732195616153 (+2 ms)Closed at 1732195616153 2024-11-21T13:26:56,155 INFO [RS_CLOSE_REGION-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed 15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,155 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=15530bc3c4bda91d823425a8a74e3630, regionState=CLOSED 2024-11-21T13:26:56,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 because future has completed 2024-11-21T13:26:56,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-21T13:26:56,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure 15530bc3c4bda91d823425a8a74e3630, server=4fe6efa1e611,33573,1732195430851 in 207 msec 2024-11-21T13:26:56,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-21T13:26:56,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=15530bc3c4bda91d823425a8a74e3630, UNASSIGN in 212 msec 2024-11-21T13:26:56,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-21T13:26:56,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 214 msec 2024-11-21T13:26:56,161 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732195616161"}]},"ts":"1732195616161"} 2024-11-21T13:26:56,162 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-21T13:26:56,162 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-21T13:26:56,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 222 msec 2024-11-21T13:26:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-21T13:26:56,259 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:56,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-21T13:26:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:56,261 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T13:26:56,262 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:56,264 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,265 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C, FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/recovered.edits] 2024-11-21T13:26:56,267 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/6d563eb5f8014f8fbf572f75737a1a51 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/6d563eb5f8014f8fbf572f75737a1a51 2024-11-21T13:26:56,268 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d9f88959b32f491fa84c4418e2c8edae to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/A/d9f88959b32f491fa84c4418e2c8edae 2024-11-21T13:26:56,269 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/bea47412f37740829cae15ec4b259b57 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/bea47412f37740829cae15ec4b259b57 2024-11-21T13:26:56,270 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/ffddf790238f44e5b374c749846e825e to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/B/ffddf790238f44e5b374c749846e825e 2024-11-21T13:26:56,271 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/013fd4feddb147128dc5c19134f051e3 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/013fd4feddb147128dc5c19134f051e3 2024-11-21T13:26:56,272 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/7773b5756080446b8db0ff43b214dd17 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/C/7773b5756080446b8db0ff43b214dd17 2024-11-21T13:26:56,273 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/recovered.edits/318.seqid to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630/recovered.edits/318.seqid 2024-11-21T13:26:56,274 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/default/TestAcidGuarantees/15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,274 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-21T13:26:56,274 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T13:26:56,274 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-21T13:26:56,276 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112102063bf65dc24393b5c39853466e9af2_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112102063bf65dc24393b5c39853466e9af2_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,277 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121030b64f5b2a34db8bdcd96c213c2c6f7_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121030b64f5b2a34db8bdcd96c213c2c6f7_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,277 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210c1351d55adf4cc2a86afea3c84dc768_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411210c1351d55adf4cc2a86afea3c84dc768_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,278 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121234c5d0faec34cb39ca2a30dde1af110_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121234c5d0faec34cb39ca2a30dde1af110_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,279 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112140bdf562b9ec4b1ba85bb4e196167c37_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112140bdf562b9ec4b1ba85bb4e196167c37_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,279 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112152bddbb598e241ee882bd53fc807f84b_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112152bddbb598e241ee882bd53fc807f84b_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,280 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215ace7d52fe9e45ba91a1e1fb5aa24109_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411215ace7d52fe9e45ba91a1e1fb5aa24109_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,281 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216e4166981ee94ff4b088d33fa8863855_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411216e4166981ee94ff4b088d33fa8863855_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,282 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121786bf86a3bbb4a3a8431f2f0c7908ee0_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121786bf86a3bbb4a3a8431f2f0c7908ee0_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,282 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219bb264804b0e4162a20f346e9c61f603_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411219bb264804b0e4162a20f346e9c61f603_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,283 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a7784db4fad4444e817c8ebaf02708f0_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121a7784db4fad4444e817c8ebaf02708f0_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,284 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cd0333e5564e40ef91cf2e6176722223_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121cd0333e5564e40ef91cf2e6176722223_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,284 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d7ad589f136447e09bdf72af2458baed_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121d7ad589f136447e09bdf72af2458baed_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,285 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e3edb5b64d434059a87bd126cd693292_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e3edb5b64d434059a87bd126cd693292_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,286 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e5dfa0ac3750408c93807a7cec1aeed7_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121e5dfa0ac3750408c93807a7cec1aeed7_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,287 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f23d0a534312452e8dbec6ebb3b93ad5_15530bc3c4bda91d823425a8a74e3630 to hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241121f23d0a534312452e8dbec6ebb3b93ad5_15530bc3c4bda91d823425a8a74e3630 2024-11-21T13:26:56,287 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-21T13:26:56,288 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:56,291 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-21T13:26:56,292 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-21T13:26:56,293 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:56,293 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-21T13:26:56,293 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732195616293"}]},"ts":"9223372036854775807"} 2024-11-21T13:26:56,294 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-21T13:26:56,294 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 15530bc3c4bda91d823425a8a74e3630, NAME => 'TestAcidGuarantees,,1732195586687.15530bc3c4bda91d823425a8a74e3630.', STARTKEY => '', ENDKEY => ''}] 2024-11-21T13:26:56,295 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-21T13:26:56,295 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732195616295"}]},"ts":"9223372036854775807"} 2024-11-21T13:26:56,296 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-21T13:26:56,296 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-21T13:26:56,297 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 36 msec 2024-11-21T13:26:56,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46031 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-21T13:26:56,369 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-21T13:26:56,370 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-21T13:26:56,377 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 236) - Thread LEAK? -, OpenFileDescriptor=451 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=429 (was 431), ProcessCount=11 (was 11), AvailableMemoryMB=2052 (was 2087) 2024-11-21T13:26:56,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T13:26:56,378 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T13:26:56,378 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T13:26:56,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:56,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:56,378 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T13:26:56,378 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T13:26:56,378 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1133720664, stopped=false 2024-11-21T13:26:56,379 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=4fe6efa1e611,46031,1732195430123 2024-11-21T13:26:56,447 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T13:26:56,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T13:26:56,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:26:56,447 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:26:56,447 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T13:26:56,448 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T13:26:56,448 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T13:26:56,448 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:56,448 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T13:26:56,448 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T13:26:56,449 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '4fe6efa1e611,33573,1732195430851' ***** 2024-11-21T13:26:56,449 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T13:26:56,449 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T13:26:56,449 INFO [RS:0;4fe6efa1e611:33573 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T13:26:56,449 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T13:26:56,449 INFO [RS:0;4fe6efa1e611:33573 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T13:26:56,449 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(959): stopping server 4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:56,450 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T13:26:56,450 INFO [RS:0;4fe6efa1e611:33573 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;4fe6efa1e611:33573. 2024-11-21T13:26:56,450 DEBUG [RS:0;4fe6efa1e611:33573 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T13:26:56,450 DEBUG [RS:0;4fe6efa1e611:33573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:56,450 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T13:26:56,450 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T13:26:56,450 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T13:26:56,450 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T13:26:56,451 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-21T13:26:56,451 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-21T13:26:56,451 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T13:26:56,451 INFO [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T13:26:56,451 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T13:26:56,451 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T13:26:56,451 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T13:26:56,451 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T13:26:56,451 INFO [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-21T13:26:56,471 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/.tmp/info/9518bca3ca91411285bd81c1d872504e is 95, key is TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8./info:/1732195462557/DeleteFamily/seqid=0 2024-11-21T13:26:56,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742513_1689 (size=6514) 2024-11-21T13:26:56,500 INFO [regionserver/4fe6efa1e611:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T13:26:56,651 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T13:26:56,851 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T13:26:56,853 INFO [regionserver/4fe6efa1e611:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T13:26:56,854 INFO [regionserver/4fe6efa1e611:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T13:26:56,875 INFO [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/.tmp/info/9518bca3ca91411285bd81c1d872504e 2024-11-21T13:26:56,892 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/.tmp/ns/c97118fbc8b743e28e5c839fc3528be8 is 93, key is TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8./ns:/1732195462519/DeleteFamily/seqid=0 2024-11-21T13:26:56,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742514_1690 (size=5906) 2024-11-21T13:26:57,052 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T13:26:57,252 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T13:26:57,295 INFO [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/.tmp/ns/c97118fbc8b743e28e5c839fc3528be8 2024-11-21T13:26:57,316 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/.tmp/rep_barrier/71237add68a447678b7a0b19df0278c7 is 102, key is TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8./rep_barrier:/1732195462519/DeleteFamily/seqid=0 2024-11-21T13:26:57,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742515_1691 (size=6025) 2024-11-21T13:26:57,452 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T13:26:57,652 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T13:26:57,719 INFO [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/.tmp/rep_barrier/71237add68a447678b7a0b19df0278c7 2024-11-21T13:26:57,742 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/.tmp/table/c51110e880da4a54af741578bc0eaa8b is 96, key is TestAcidGuarantees,,1732195435078.78998c23688e9623a40dfb78a0d6fff8./table:/1732195462519/DeleteFamily/seqid=0 2024-11-21T13:26:57,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742516_1692 (size=5936) 2024-11-21T13:26:57,852 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T13:26:58,053 DEBUG [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-21T13:26:58,145 INFO [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/.tmp/table/c51110e880da4a54af741578bc0eaa8b 2024-11-21T13:26:58,149 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/.tmp/info/9518bca3ca91411285bd81c1d872504e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/info/9518bca3ca91411285bd81c1d872504e 2024-11-21T13:26:58,152 INFO [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/info/9518bca3ca91411285bd81c1d872504e, entries=12, sequenceid=90, filesize=6.4 K 2024-11-21T13:26:58,153 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/.tmp/ns/c97118fbc8b743e28e5c839fc3528be8 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/ns/c97118fbc8b743e28e5c839fc3528be8 2024-11-21T13:26:58,156 INFO [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/ns/c97118fbc8b743e28e5c839fc3528be8, entries=8, sequenceid=90, filesize=5.8 K 2024-11-21T13:26:58,157 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/.tmp/rep_barrier/71237add68a447678b7a0b19df0278c7 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/rep_barrier/71237add68a447678b7a0b19df0278c7 2024-11-21T13:26:58,159 INFO [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/rep_barrier/71237add68a447678b7a0b19df0278c7, entries=6, sequenceid=90, filesize=5.9 K 2024-11-21T13:26:58,159 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/.tmp/table/c51110e880da4a54af741578bc0eaa8b as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/table/c51110e880da4a54af741578bc0eaa8b 2024-11-21T13:26:58,162 INFO [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/table/c51110e880da4a54af741578bc0eaa8b, entries=7, sequenceid=90, filesize=5.8 K 2024-11-21T13:26:58,163 INFO [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1712ms, sequenceid=90, compaction requested=false 2024-11-21T13:26:58,167 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-21T13:26:58,167 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T13:26:58,167 INFO [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T13:26:58,167 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732195616451Running coprocessor pre-close hooks at 1732195616451Disabling compacts and flushes for region at 1732195616451Disabling writes for close at 1732195616451Obtaining lock to block concurrent updates at 1732195616451Preparing flush snapshotting stores in 1588230740 at 1732195616451Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732195616451Flushing stores of hbase:meta,,1.1588230740 at 1732195616452 (+1 ms)Flushing 1588230740/info: creating writer at 1732195616452Flushing 1588230740/info: appending metadata at 1732195616471 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732195616471Flushing 1588230740/ns: creating writer at 1732195616879 (+408 ms)Flushing 1588230740/ns: appending metadata at 1732195616892 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1732195616892Flushing 1588230740/rep_barrier: creating writer at 1732195617298 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732195617315 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732195617315Flushing 1588230740/table: creating writer at 1732195617722 (+407 ms)Flushing 1588230740/table: appending metadata at 1732195617741 (+19 ms)Flushing 1588230740/table: closing flushed file at 1732195617741Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@64d8574b: reopening flushed file at 1732195618148 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d6a6585: reopening flushed file at 1732195618152 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ccc44ac: reopening flushed file at 1732195618156 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@457d15cb: reopening flushed file at 1732195618159 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1712ms, sequenceid=90, compaction requested=false at 1732195618163 (+4 ms)Writing region close event to WAL at 1732195618164 (+1 ms)Running coprocessor post-close hooks at 1732195618167 (+3 ms)Closed at 1732195618167 2024-11-21T13:26:58,167 DEBUG [RS_CLOSE_META-regionserver/4fe6efa1e611:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T13:26:58,253 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(976): stopping server 4fe6efa1e611,33573,1732195430851; all regions closed. 2024-11-21T13:26:58,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741834_1010 (size=25344) 2024-11-21T13:26:58,258 DEBUG [RS:0;4fe6efa1e611:33573 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/oldWALs 2024-11-21T13:26:58,258 INFO [RS:0;4fe6efa1e611:33573 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4fe6efa1e611%2C33573%2C1732195430851.meta:.meta(num 1732195434481) 2024-11-21T13:26:58,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741833_1009 (size=15328184) 2024-11-21T13:26:58,261 DEBUG [RS:0;4fe6efa1e611:33573 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/oldWALs 2024-11-21T13:26:58,261 INFO [RS:0;4fe6efa1e611:33573 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 4fe6efa1e611%2C33573%2C1732195430851:(num 1732195434010) 2024-11-21T13:26:58,261 DEBUG [RS:0;4fe6efa1e611:33573 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T13:26:58,261 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T13:26:58,261 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T13:26:58,261 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.ChoreService(370): Chore service for: regionserver/4fe6efa1e611:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T13:26:58,262 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T13:26:58,262 INFO [regionserver/4fe6efa1e611:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T13:26:58,262 INFO [RS:0;4fe6efa1e611:33573 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33573 2024-11-21T13:26:58,321 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/4fe6efa1e611,33573,1732195430851 2024-11-21T13:26:58,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T13:26:58,321 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T13:26:58,323 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [4fe6efa1e611,33573,1732195430851] 2024-11-21T13:26:58,346 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/4fe6efa1e611,33573,1732195430851 already deleted, retry=false 2024-11-21T13:26:58,346 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 4fe6efa1e611,33573,1732195430851 expired; onlineServers=0 2024-11-21T13:26:58,347 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '4fe6efa1e611,46031,1732195430123' ***** 2024-11-21T13:26:58,347 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T13:26:58,347 INFO [M:0;4fe6efa1e611:46031 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T13:26:58,347 INFO [M:0;4fe6efa1e611:46031 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T13:26:58,347 DEBUG [M:0;4fe6efa1e611:46031 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T13:26:58,347 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T13:26:58,347 DEBUG [M:0;4fe6efa1e611:46031 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T13:26:58,347 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster-HFileCleaner.small.0-1732195433604 {}] cleaner.HFileCleaner(306): Exit Thread[master/4fe6efa1e611:0:becomeActiveMaster-HFileCleaner.small.0-1732195433604,5,FailOnTimeoutGroup] 2024-11-21T13:26:58,347 DEBUG [master/4fe6efa1e611:0:becomeActiveMaster-HFileCleaner.large.0-1732195433600 {}] cleaner.HFileCleaner(306): Exit Thread[master/4fe6efa1e611:0:becomeActiveMaster-HFileCleaner.large.0-1732195433600,5,FailOnTimeoutGroup] 2024-11-21T13:26:58,347 INFO [M:0;4fe6efa1e611:46031 {}] hbase.ChoreService(370): Chore service for: master/4fe6efa1e611:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T13:26:58,348 INFO [M:0;4fe6efa1e611:46031 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T13:26:58,348 DEBUG [M:0;4fe6efa1e611:46031 {}] master.HMaster(1795): Stopping service threads 2024-11-21T13:26:58,348 INFO [M:0;4fe6efa1e611:46031 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T13:26:58,348 INFO [M:0;4fe6efa1e611:46031 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T13:26:58,348 ERROR [M:0;4fe6efa1e611:46031 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-21T13:26:58,349 INFO [M:0;4fe6efa1e611:46031 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T13:26:58,349 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T13:26:58,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T13:26:58,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T13:26:58,357 DEBUG [M:0;4fe6efa1e611:46031 {}] zookeeper.ZKUtil(347): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T13:26:58,357 WARN [M:0;4fe6efa1e611:46031 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T13:26:58,358 INFO [M:0;4fe6efa1e611:46031 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/.lastflushedseqids 2024-11-21T13:26:58,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742517_1693 (size=72) 2024-11-21T13:26:58,437 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T13:26:58,437 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33573-0x1015d90a7550001, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T13:26:58,438 INFO [RS:0;4fe6efa1e611:33573 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T13:26:58,438 INFO [RS:0;4fe6efa1e611:33573 {}] regionserver.HRegionServer(1031): Exiting; stopping=4fe6efa1e611,33573,1732195430851; zookeeper connection closed. 2024-11-21T13:26:58,439 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@615e79d8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@615e79d8 2024-11-21T13:26:58,440 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-21T13:26:58,773 INFO [M:0;4fe6efa1e611:46031 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T13:26:58,774 INFO [M:0;4fe6efa1e611:46031 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T13:26:58,774 DEBUG [M:0;4fe6efa1e611:46031 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T13:26:58,774 INFO [M:0;4fe6efa1e611:46031 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T13:26:58,774 DEBUG [M:0;4fe6efa1e611:46031 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T13:26:58,774 DEBUG [M:0;4fe6efa1e611:46031 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T13:26:58,775 DEBUG [M:0;4fe6efa1e611:46031 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T13:26:58,775 INFO [M:0;4fe6efa1e611:46031 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=784.24 KB heapSize=966.36 KB 2024-11-21T13:26:58,798 DEBUG [M:0;4fe6efa1e611:46031 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/44a072b561a64c918253712bcf3a4d83 is 82, key is hbase:meta,,1/info:regioninfo/1732195434601/Put/seqid=0 2024-11-21T13:26:58,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742518_1694 (size=5672) 2024-11-21T13:26:59,203 INFO [M:0;4fe6efa1e611:46031 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2257 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/44a072b561a64c918253712bcf3a4d83 2024-11-21T13:26:59,235 DEBUG [M:0;4fe6efa1e611:46031 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a5fc32a51b824ca0863b224db1bebe0e is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x95/proc:d/1732195589752/Put/seqid=0 2024-11-21T13:26:59,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742519_1695 (size=44473) 2024-11-21T13:26:59,638 INFO [M:0;4fe6efa1e611:46031 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=783.69 KB at sequenceid=2257 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a5fc32a51b824ca0863b224db1bebe0e 2024-11-21T13:26:59,642 INFO [M:0;4fe6efa1e611:46031 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a5fc32a51b824ca0863b224db1bebe0e 2024-11-21T13:26:59,660 DEBUG [M:0;4fe6efa1e611:46031 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a893771bf3814679a902ffb44b9e1dfb is 69, key is 4fe6efa1e611,33573,1732195430851/rs:state/1732195433694/Put/seqid=0 2024-11-21T13:26:59,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073742520_1696 (size=5156) 2024-11-21T13:27:00,065 INFO [M:0;4fe6efa1e611:46031 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2257 (bloomFilter=true), to=hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a893771bf3814679a902ffb44b9e1dfb 2024-11-21T13:27:00,076 DEBUG [M:0;4fe6efa1e611:46031 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/44a072b561a64c918253712bcf3a4d83 as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/44a072b561a64c918253712bcf3a4d83 2024-11-21T13:27:00,080 INFO [M:0;4fe6efa1e611:46031 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/44a072b561a64c918253712bcf3a4d83, entries=8, sequenceid=2257, filesize=5.5 K 2024-11-21T13:27:00,081 DEBUG [M:0;4fe6efa1e611:46031 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a5fc32a51b824ca0863b224db1bebe0e as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a5fc32a51b824ca0863b224db1bebe0e 2024-11-21T13:27:00,084 INFO [M:0;4fe6efa1e611:46031 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a5fc32a51b824ca0863b224db1bebe0e 2024-11-21T13:27:00,085 INFO [M:0;4fe6efa1e611:46031 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a5fc32a51b824ca0863b224db1bebe0e, entries=172, sequenceid=2257, filesize=43.4 K 2024-11-21T13:27:00,085 DEBUG [M:0;4fe6efa1e611:46031 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a893771bf3814679a902ffb44b9e1dfb as hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a893771bf3814679a902ffb44b9e1dfb 2024-11-21T13:27:00,089 INFO [M:0;4fe6efa1e611:46031 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39169/user/jenkins/test-data/63b1ee00-a624-fad2-d4af-d5078cf5ec48/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a893771bf3814679a902ffb44b9e1dfb, entries=1, sequenceid=2257, filesize=5.0 K 2024-11-21T13:27:00,090 INFO [M:0;4fe6efa1e611:46031 {}] regionserver.HRegion(3140): Finished flush of dataSize ~784.24 KB/803064, heapSize ~966.06 KB/989248, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1315ms, sequenceid=2257, compaction requested=false 2024-11-21T13:27:00,091 INFO [M:0;4fe6efa1e611:46031 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T13:27:00,091 DEBUG [M:0;4fe6efa1e611:46031 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732195618774Disabling compacts and flushes for region at 1732195618774Disabling writes for close at 1732195618775 (+1 ms)Obtaining lock to block concurrent updates at 1732195618775Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732195618775Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=803064, getHeapSize=989488, getOffHeapSize=0, getCellsCount=2454 at 1732195618776 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732195618777 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732195618777Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732195618798 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732195618798Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732195619212 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732195619234 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732195619234Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732195619642 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732195619660 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732195619660Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@67bd1414: reopening flushed file at 1732195620075 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19eaef38: reopening flushed file at 1732195620080 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@459775cd: reopening flushed file at 1732195620085 (+5 ms)Finished flush of dataSize ~784.24 KB/803064, heapSize ~966.06 KB/989248, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1315ms, sequenceid=2257, compaction requested=false at 1732195620090 (+5 ms)Writing region close event to WAL at 1732195620091 (+1 ms)Closed at 1732195620091 2024-11-21T13:27:00,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:44249 is added to blk_1073741830_1006 (size=950976) 2024-11-21T13:27:00,095 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T13:27:00,095 INFO [M:0;4fe6efa1e611:46031 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T13:27:00,095 INFO [M:0;4fe6efa1e611:46031 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46031 2024-11-21T13:27:00,095 INFO [M:0;4fe6efa1e611:46031 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T13:27:00,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T13:27:00,221 INFO [M:0;4fe6efa1e611:46031 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T13:27:00,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46031-0x1015d90a7550000, quorum=127.0.0.1:59696, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T13:27:00,234 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1e531827{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T13:27:00,237 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4ea205b6{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T13:27:00,237 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T13:27:00,237 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@24457f8e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T13:27:00,237 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@350067e1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/hadoop.log.dir/,STOPPED} 2024-11-21T13:27:00,240 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T13:27:00,240 WARN [BP-814699736-172.17.0.2-1732195426084 heartbeating to localhost/127.0.0.1:39169 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T13:27:00,240 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T13:27:00,240 WARN [BP-814699736-172.17.0.2-1732195426084 heartbeating to localhost/127.0.0.1:39169 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-814699736-172.17.0.2-1732195426084 (Datanode Uuid 47d5bed9-0627-49b8-995c-9e5afce33aad) service to localhost/127.0.0.1:39169 2024-11-21T13:27:00,242 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/cluster_cd6fb7cf-33aa-fbb2-dd15-3e00518ea929/data/data1/current/BP-814699736-172.17.0.2-1732195426084 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T13:27:00,242 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/cluster_cd6fb7cf-33aa-fbb2-dd15-3e00518ea929/data/data2/current/BP-814699736-172.17.0.2-1732195426084 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T13:27:00,243 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T13:27:00,248 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7f405bdd{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T13:27:00,249 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5b829fcb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T13:27:00,249 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T13:27:00,249 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@31a6a643{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T13:27:00,249 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@610d6b27{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6673393d-47e4-e42f-3465-8b9a61979ac5/hadoop.log.dir/,STOPPED} 2024-11-21T13:27:00,264 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T13:27:00,387 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down 2024-11-21T13:27:00,469 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T13:27:00,469 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-21T13:27:00,469 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-21T13:27:00,470 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees